From 3ab1381de8277f00e23cc840fe4fa11c644a508b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 27 Apr 2022 18:06:27 +0200 Subject: [PATCH 01/46] Refactor the `LocatorService` implementations and add `locate(Link)` --- Makefile | 5 ++ .../Locator/DefaultLocatorService.swift | 67 ++++++++++++++----- .../Services/Locator/LocatorService.swift | 24 ++++--- .../Positions/InMemoryPositionsService.swift | 23 +++++++ .../PerResourcePositionsService.swift | 12 +--- .../Services/PublicationServicesBuilder.swift | 2 +- Sources/Shared/Toolkit/Weak.swift | 24 ++++++- .../Audio/Services/AudioLocatorService.swift | 55 ++++++--------- TestApp/.gitignore | 2 + .../Locator/DefaultLocatorServiceTests.swift | 8 ++- .../Services/AudioLocatorServiceTests.swift | 4 +- 11 files changed, 151 insertions(+), 75 deletions(-) create mode 100644 Sources/Shared/Publication/Services/Positions/InMemoryPositionsService.swift diff --git a/Makefile b/Makefile index 18ef3ac3a..2ae3ae81e 100644 --- a/Makefile +++ b/Makefile @@ -16,3 +16,8 @@ scripts: yarn --cwd "$(SCRIPTS_PATH)" run format yarn --cwd "$(SCRIPTS_PATH)" run lint yarn --cwd "$(SCRIPTS_PATH)" run bundle + +.PHONY: test +test: + xcodebuild test -scheme "Readium-Package" -destination "platform=iOS Simulator,name=iPhone 12" + diff --git a/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift b/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift index 4f98b21bb..cb87bb3cf 100644 --- a/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift +++ b/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift @@ -8,35 +8,70 @@ import Foundation /// A default implementation of the `LocatorService` using the `PositionsService` to locate its inputs. open class DefaultLocatorService: LocatorService, Loggable { - - let readingOrder: [Link] - let positionsByReadingOrder: () -> [[Locator]] - - public init(readingOrder: [Link], positionsByReadingOrder: @escaping () -> [[Locator]]) { - self.readingOrder = readingOrder - self.positionsByReadingOrder = positionsByReadingOrder - } - public convenience init(readingOrder: [Link], publication: Weak) { - self.init(readingOrder: readingOrder, positionsByReadingOrder: { publication()?.positionsByReadingOrder ?? [] }) + public let publication: Weak + + public init(publication: Weak) { + self.publication = publication } + /// Locates the target of the given `locator`. + /// + /// If `locator.href` can be found in the links, `locator` will be returned directly. + /// Otherwise, will attempt to find the closest match using `totalProgression`, `position`, + /// `fragments`, etc. open func locate(_ locator: Locator) -> Locator? { - guard readingOrder.firstIndex(withHREF: locator.href) != nil else { + guard let publication = publication() else { return nil } - - return locator + + if publication.link(withHREF: locator.href) != nil { + return locator + } + + if let totalProgression = locator.locations.totalProgression, let target = locate(progression: totalProgression) { + return target.copy( + title: locator.title, + text: { $0 = locator.text } + ) + } + + return nil } - + + open func locate(_ link: Link) -> Locator? { + let components = link.href.split(separator: "#", maxSplits: 1).map(String.init) + let href = components.first ?? link.href + let fragment = components.getOrNil(1) + + guard + let resourceLink = publication()?.link(withHREF: href), + let type = resourceLink.type + else { + return nil + } + + return Locator( + href: href, + type: type, + title: resourceLink.title ?? link.title, + locations: Locator.Locations( + fragments: Array(ofNotNil: fragment), + progression: (fragment != nil) ? 0.0 : nil + ) + ) + } + open func locate(progression totalProgression: Double) -> Locator? { guard 0.0...1.0 ~= totalProgression else { log(.error, "Progression must be between 0.0 and 1.0, received \(totalProgression)") return nil } - let positions = positionsByReadingOrder() - guard let (readingOrderIndex, position) = findClosest(to: totalProgression, in: positions) else { + guard + let positions = publication()?.positionsByReadingOrder, + let (readingOrderIndex, position) = findClosest(to: totalProgression, in: positions) + else { return nil } diff --git a/Sources/Shared/Publication/Services/Locator/LocatorService.swift b/Sources/Shared/Publication/Services/Locator/LocatorService.swift index 4d076a1a5..1cd7bfaaa 100644 --- a/Sources/Shared/Publication/Services/Locator/LocatorService.swift +++ b/Sources/Shared/Publication/Services/Locator/LocatorService.swift @@ -16,13 +16,21 @@ public typealias LocatorServiceFactory = (PublicationServiceContext) -> LocatorS /// - Converting a `Locator` which was created from an alternate manifest with a different reading /// order. For example, when downloading a streamed manifest or offloading a package. public protocol LocatorService: PublicationService { - + /// Locates the target of the given `locator`. func locate(_ locator: Locator) -> Locator? - + + /// Locates the target of the given `link`. + func locate(_ link: Link) -> Locator? + /// Locates the target at the given `progression` relative to the whole publication. func locate(progression: Double) -> Locator? - +} + +public extension LocatorService { + func locate(_ locator: Locator) -> Locator? { nil } + func locate(_ link: Link) -> Locator? { nil } + func locate(progression: Double) -> Locator? { nil } } @@ -31,19 +39,19 @@ public protocol LocatorService: PublicationService { public extension Publication { /// Locates the target of the given `locator`. - /// - /// If `locator.href` can be found in the reading order, `locator` will be returned directly. - /// Otherwise, will attempt to find the closest match using `totalProgression`, `position`, - /// `fragments`, etc. func locate(_ locator: Locator) -> Locator? { findService(LocatorService.self)?.locate(locator) } - + /// Locates the target at the given `progression` relative to the whole publication. func locate(progression: Double) -> Locator? { findService(LocatorService.self)?.locate(progression: progression) } + /// Locates the target of the given `link`. + func locate(_ link: Link) -> Locator? { + findService(LocatorService.self)?.locate(link) + } } diff --git a/Sources/Shared/Publication/Services/Positions/InMemoryPositionsService.swift b/Sources/Shared/Publication/Services/Positions/InMemoryPositionsService.swift new file mode 100644 index 000000000..e99b0e4eb --- /dev/null +++ b/Sources/Shared/Publication/Services/Positions/InMemoryPositionsService.swift @@ -0,0 +1,23 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// A [PositionsService] holding the pre-computed position locators in memory. +public class InMemoryPositionsService : PositionsService { + + public private(set) var positionsByReadingOrder: [[Locator]] + + public init(positionsByReadingOrder: [[Locator]]) { + self.positionsByReadingOrder = positionsByReadingOrder + } + + public static func makeFactory(positionsByReadingOrder: [[Locator]]) -> (PublicationServiceContext) -> InMemoryPositionsService { + { _ in + InMemoryPositionsService(positionsByReadingOrder: positionsByReadingOrder) + } + } +} diff --git a/Sources/Shared/Publication/Services/Positions/PerResourcePositionsService.swift b/Sources/Shared/Publication/Services/Positions/PerResourcePositionsService.swift index 49a40d272..055c39b9d 100644 --- a/Sources/Shared/Publication/Services/Positions/PerResourcePositionsService.swift +++ b/Sources/Shared/Publication/Services/Positions/PerResourcePositionsService.swift @@ -1,12 +1,7 @@ // -// PerResourcePositionsService.swift -// r2-shared-swift -// -// Created by Mickaël Menu on 01/06/2020. -// -// Copyright 2020 Readium Foundation. All rights reserved. -// Use of this source code is governed by a BSD-style license which is detailed -// in the LICENSE file present in the project repository where this source code is maintained. +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. // import Foundation @@ -46,6 +41,5 @@ public final class PerResourcePositionsService: PositionsService { PerResourcePositionsService(readingOrder: context.manifest.readingOrder, fallbackMediaType: fallbackMediaType) } } - } diff --git a/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift b/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift index 8e3e01353..2004e4533 100644 --- a/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift +++ b/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift @@ -21,7 +21,7 @@ public struct PublicationServicesBuilder { public init( contentProtection: ContentProtectionServiceFactory? = nil, cover: CoverServiceFactory? = nil, - locator: LocatorServiceFactory? = { DefaultLocatorService(readingOrder: $0.manifest.readingOrder, publication: $0.publication) }, + locator: LocatorServiceFactory? = { DefaultLocatorService(publication: $0.publication) }, positions: PositionsServiceFactory? = nil, search: SearchServiceFactory? = nil, setup: (inout PublicationServicesBuilder) -> Void = { _ in } diff --git a/Sources/Shared/Toolkit/Weak.swift b/Sources/Shared/Toolkit/Weak.swift index e641762b2..362ac82a0 100644 --- a/Sources/Shared/Toolkit/Weak.swift +++ b/Sources/Shared/Toolkit/Weak.swift @@ -11,7 +11,7 @@ import Foundation /// Get the reference by calling `weakVar()`. /// Conveniently, the reference can be reset by setting the `ref` property. @dynamicCallable -public final class Weak { +public class Weak { // Weakly held reference. public weak var ref: T? @@ -23,4 +23,24 @@ public final class Weak { public func dynamicallyCall(withArguments args: [Any]) -> T? { ref } -} \ No newline at end of file +} + +/// Smart pointer passing as a Weak reference but preventing the reference from being lost. +/// Mainly useful for the unit test suite. +public class _Strong: Weak { + + private var strongRef: T? + + public override var ref: T? { + get { super.ref } + set { + super.ref = newValue + strongRef = newValue + } + } + + public override init(_ ref: T? = nil) { + self.strongRef = ref + super.init(ref) + } +} diff --git a/Sources/Streamer/Parser/Audio/Services/AudioLocatorService.swift b/Sources/Streamer/Parser/Audio/Services/AudioLocatorService.swift index 63e1ccb6d..75edb17da 100644 --- a/Sources/Streamer/Parser/Audio/Services/AudioLocatorService.swift +++ b/Sources/Streamer/Parser/Audio/Services/AudioLocatorService.swift @@ -8,39 +8,26 @@ import Foundation import R2Shared /// Locator service for audio publications. -final class AudioLocatorService: LocatorService { - - /// Total duration of the publication. - private let totalDuration: Double? - +final class AudioLocatorService: DefaultLocatorService { + + static func makeFactory() -> (PublicationServiceContext) -> AudioLocatorService { + { context in AudioLocatorService(publication: context.publication) } + } + + private lazy var readingOrder: [Link] = + publication()?.readingOrder ?? [] + /// Duration per reading order index. - private let durations: [Double] - - private let readingOrder: [Link] - - init(readingOrder: [Link]) { - self.durations = readingOrder.map { $0.duration ?? 0 } + private lazy var durations: [Double] = + readingOrder.map { $0.duration ?? 0 } + + /// Total duration of the publication. + private lazy var totalDuration: Double? = { let totalDuration = durations.reduce(0, +) - self.totalDuration = (totalDuration > 0) ? totalDuration : nil - self.readingOrder = readingOrder - } - - func locate(_ locator: Locator) -> Locator? { - if readingOrder.firstIndex(withHREF: locator.href) != nil { - return locator - } - - if let totalProgression = locator.locations.totalProgression, let target = locate(progression: totalProgression) { - return target.copy( - title: locator.title, - text: { $0 = locator.text } - ) - } - - return nil - } - - func locate(progression: Double) -> Locator? { + return (totalDuration > 0) ? totalDuration : nil + }() + + override func locate(progression: Double) -> Locator? { guard let totalDuration = totalDuration else { return nil } @@ -68,11 +55,7 @@ final class AudioLocatorService: LocatorService { ) ) } - - static func makeFactory() -> (PublicationServiceContext) -> AudioLocatorService { - { context in AudioLocatorService(readingOrder: context.manifest.readingOrder) } - } - + /// Finds the reading order item containing the time `position` (in seconds), as well as its /// start time. private func readingOrderItemAtPosition(_ position: Double) -> (link: Link, startPosition: Double)? { diff --git a/TestApp/.gitignore b/TestApp/.gitignore index 8225e5fbf..3c74969de 100644 --- a/TestApp/.gitignore +++ b/TestApp/.gitignore @@ -1,6 +1,8 @@ TestApp.xcodeproj TestApp.xcworkspace project.yml +# IntelliJ AppCode +.idea Carthage/ Cartfile diff --git a/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift b/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift index 9fd681d1f..d880d0633 100644 --- a/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift +++ b/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift @@ -114,9 +114,13 @@ class DefaultLocatorServiceTests: XCTestCase { } func makeService(readingOrder: [Link] = [], positions: [[Locator]] = []) -> DefaultLocatorService { - DefaultLocatorService(readingOrder: readingOrder, positionsByReadingOrder: { positions }) + DefaultLocatorService(publication: _Strong(Publication( + manifest: Manifest(metadata: Metadata(title: ""), readingOrder: readingOrder), + servicesBuilder: PublicationServicesBuilder( + positions: InMemoryPositionsService.makeFactory(positionsByReadingOrder: positions) + ) + ))) } - } private let positionsFixture: [[Locator]] = [ diff --git a/Tests/StreamerTests/Parser/Audio/Services/AudioLocatorServiceTests.swift b/Tests/StreamerTests/Parser/Audio/Services/AudioLocatorServiceTests.swift index 3f9825973..93dc64dcc 100644 --- a/Tests/StreamerTests/Parser/Audio/Services/AudioLocatorServiceTests.swift +++ b/Tests/StreamerTests/Parser/Audio/Services/AudioLocatorServiceTests.swift @@ -165,7 +165,9 @@ class AudioLocatorServiceTests: XCTestCase { private func makeService(readingOrder: [Link]) -> AudioLocatorService { AudioLocatorService( - readingOrder: readingOrder + publication: _Strong(Publication( + manifest: Manifest(metadata: Metadata(title: ""), readingOrder: readingOrder) + )) ) } From 69640753d77b126a5b9a904d8ea0671caf4d0756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 27 Apr 2022 18:34:52 +0200 Subject: [PATCH 02/46] Add tests and deprecate `Locator(link: Link)` --- Makefile | 3 +- Sources/Shared/Publication/Locator.swift | 3 +- Sources/Shared/Publication/Manifest.swift | 31 +++++++- Sources/Shared/Publication/Publication.swift | 36 ++------- .../Locator/DefaultLocatorService.swift | 2 +- .../Services/Search/StringSearchService.swift | 10 ++- .../Common/Outline/OutlineTableView.swift | 17 ++-- .../Locator/DefaultLocatorServiceTests.swift | 78 ++++++++++++++++++- 8 files changed, 134 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 2ae3ae81e..cbbe29f95 100644 --- a/Makefile +++ b/Makefile @@ -19,5 +19,6 @@ scripts: .PHONY: test test: - xcodebuild test -scheme "Readium-Package" -destination "platform=iOS Simulator,name=iPhone 12" + # To limit to a particular test suite: -only-testing:R2SharedTests + xcodebuild test -scheme "Readium-Package" -destination "platform=iOS Simulator,name=iPhone 12" | xcbeautify -q diff --git a/Sources/Shared/Publication/Locator.swift b/Sources/Shared/Publication/Locator.swift index b787b0216..4d57543cb 100644 --- a/Sources/Shared/Publication/Locator.swift +++ b/Sources/Shared/Publication/Locator.swift @@ -70,7 +70,8 @@ public struct Locator: Hashable, CustomStringConvertible, Loggable { try self.init(json: json, warnings: warnings) } - + + @available(*, deprecated, message: "This may create an incorrect `Locator` if the link `type` is missing. Use `publication.locate(Link)` instead.") public init(link: Link) { let components = link.href.split(separator: "#", maxSplits: 1).map(String.init) let fragments = (components.count > 1) ? [String(components[1])] : [] diff --git a/Sources/Shared/Publication/Manifest.swift b/Sources/Shared/Publication/Manifest.swift index afd0c8b15..e3287ba96 100644 --- a/Sources/Shared/Publication/Manifest.swift +++ b/Sources/Shared/Publication/Manifest.swift @@ -126,7 +126,36 @@ public struct Manifest: JSONEquatable, Hashable { return metadata.conformsTo.contains(profile) } - + + /// Finds the first Link having the given `href` in the manifest's links. + public func link(withHREF href: String) -> Link? { + func deepFind(in linkLists: [Link]...) -> Link? { + for links in linkLists { + for link in links { + if link.href == href { + return link + } else if let child = deepFind(in: link.alternates, link.children) { + return child + } + } + } + + return nil + } + + var link = deepFind(in: readingOrder, resources, links) + if + link == nil, + let shortHREF = href.components(separatedBy: .init(charactersIn: "#?")).first, + shortHREF != href + { + // Tries again, but without the anchor and query parameters. + link = self.link(withHREF: shortHREF) + } + + return link + } + /// Finds the first link with the given relation in the manifest's links. public func link(withRel rel: LinkRelation) -> Link? { return readingOrder.first(withRel: rel) diff --git a/Sources/Shared/Publication/Publication.swift b/Sources/Shared/Publication/Publication.swift index dc55f15b6..74f7e60db 100644 --- a/Sources/Shared/Publication/Publication.swift +++ b/Sources/Shared/Publication/Publication.swift @@ -84,7 +84,7 @@ public class Publication: Loggable { /// Returns whether this publication conforms to the given Readium Web Publication Profile. public func conforms(to profile: Profile) -> Bool { - return manifest.conforms(to: profile) + manifest.conforms(to: profile) } /// The URL where this publication is served, computed from the `Link` with `self` relation. @@ -97,41 +97,17 @@ public class Publication: Loggable { /// Finds the first Link having the given `href` in the publication's links. public func link(withHREF href: String) -> Link? { - func deepFind(in linkLists: [Link]...) -> Link? { - for links in linkLists { - for link in links { - if link.href == href { - return link - } else if let child = deepFind(in: link.alternates, link.children) { - return child - } - } - } - - return nil - } - - var link = deepFind(in: readingOrder, resources, links) - if - link == nil, - let shortHREF = href.components(separatedBy: .init(charactersIn: "#?")).first, - shortHREF != href - { - // Tries again, but without the anchor and query parameters. - link = self.link(withHREF: shortHREF) - } - - return link + manifest.link(withHREF: href) } /// Finds the first link with the given relation in the publication's links. public func link(withRel rel: LinkRelation) -> Link? { - return manifest.link(withRel: rel) + manifest.link(withRel: rel) } /// Finds all the links with the given relation in the publication's links. public func links(withRel rel: LinkRelation) -> [Link] { - return manifest.links(withRel: rel) + manifest.links(withRel: rel) } /// Returns the resource targeted by the given `link`. @@ -161,12 +137,12 @@ public class Publication: Loggable { /// /// e.g. `findService(PositionsService.self)` public func findService(_ serviceType: T.Type) -> T? { - return services.first { $0 is T } as? T + services.first { $0 is T } as? T } /// Finds all the services implementing the given service type. public func findServices(_ serviceType: T.Type) -> [T] { - return services.filter { $0 is T } as! [T] + services.filter { $0 is T } as! [T] } /// Sets the URL where this `Publication`'s RWPM manifest is served. diff --git a/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift b/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift index cb87bb3cf..026812efa 100644 --- a/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift +++ b/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift @@ -57,7 +57,7 @@ open class DefaultLocatorService: LocatorService, Loggable { title: resourceLink.title ?? link.title, locations: Locator.Locations( fragments: Array(ofNotNil: fragment), - progression: (fragment != nil) ? 0.0 : nil + progression: (fragment == nil) ? 0.0 : nil ) ) } diff --git a/Sources/Shared/Publication/Services/Search/StringSearchService.swift b/Sources/Shared/Publication/Services/Search/StringSearchService.swift index aa36b8a7d..fff15be4b 100644 --- a/Sources/Shared/Publication/Services/Search/StringSearchService.swift +++ b/Sources/Shared/Publication/Services/Search/StringSearchService.swift @@ -154,13 +154,17 @@ public class _StringSearchService: _SearchService { } private func findLocators(in link: Link, resourceIndex: Int, text: String, cancellable: CancellableObject) -> [Locator] { - guard !text.isEmpty else { + guard + !text.isEmpty, + var resourceLocator = publication.locate(link) + else { return [] } let currentLocale = options.language.map { Locale(identifier: $0) } ?? locale - let title = publication.tableOfContents.titleMatchingHREF(link.href) ?? link.title - let resourceLocator = Locator(link: link).copy(title: title) + resourceLocator = resourceLocator.copy( + title: publication.tableOfContents.titleMatchingHREF(link.href) ?? link.title + ) var locators: [Locator] = [] diff --git a/TestApp/Sources/Reader/Common/Outline/OutlineTableView.swift b/TestApp/Sources/Reader/Common/Outline/OutlineTableView.swift index e0c08d1ea..fc5276ac1 100644 --- a/TestApp/Sources/Reader/Common/Outline/OutlineTableView.swift +++ b/TestApp/Sources/Reader/Common/Outline/OutlineTableView.swift @@ -19,27 +19,28 @@ enum OutlineSection: Int { } struct OutlineTableView: View { + private let publication: Publication @ObservedObject private var bookmarksModel: BookmarksViewModel @ObservedObject private var highlightsModel: HighlightsViewModel @State private var selectedSection: OutlineSection = .tableOfContents // Outlines (list of links) to display for each section. private var outlines: [OutlineSection: [(level: Int, link: R2Shared.Link)]] = [:] - + init(publication: Publication, bookId: Book.Id, bookmarkRepository: BookmarkRepository, highlightRepository: HighlightRepository) { - + self.publication = publication + self.bookmarksModel = BookmarksViewModel(bookId: bookId, repository: bookmarkRepository) + self.highlightsModel = HighlightsViewModel(bookId: bookId, repository: highlightRepository) + func flatten(_ links: [R2Shared.Link], level: Int = 0) -> [(level: Int, link: R2Shared.Link)] { return links.flatMap { [(level, $0)] + flatten($0.children, level: level + 1) } } - outlines = [ + self.outlines = [ .tableOfContents: flatten(publication.tableOfContents), .landmarks: flatten(publication.landmarks), .pageList: flatten(publication.pageList) ] - - bookmarksModel = BookmarksViewModel(bookId: bookId, repository: bookmarkRepository) - highlightsModel = HighlightsViewModel(bookId: bookId, repository: highlightRepository) } var body: some View { @@ -55,7 +56,9 @@ struct OutlineTableView: View { .frame(maxWidth: .infinity, alignment: .leading) .contentShape(Rectangle()) .onTapGesture { - locatorSubject.send(Locator(link: item.link)) + if let locator = publication.locate(item.link) { + locatorSubject.send(locator) + } } } } else { diff --git a/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift b/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift index d880d0633..a0ed65605 100644 --- a/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift +++ b/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift @@ -113,9 +113,83 @@ class DefaultLocatorServiceTests: XCTestCase { XCTAssertNil(service.locate(progression: 0.5)) } - func makeService(readingOrder: [Link] = [], positions: [[Locator]] = []) -> DefaultLocatorService { + func testFromMinimalLink() { + let service = makeService(readingOrder: [ + Link(href: "/href", type: "text/html", title: "Resource") + ]) + + XCTAssertEqual( + service.locate(Link(href: "/href")), + Locator(href: "/href", type: "text/html", title: "Resource", locations: Locator.Locations(progression: 0.0)) + ) + } + + func testFromLinkInReadingOrderResourcesOrLinks() { + let service = makeService( + links: [Link(href: "/href3", type: "text/html")], + readingOrder: [Link(href: "/href1", type: "text/html")], + resources: [Link(href: "/href2", type: "text/html")] + ) + + XCTAssertEqual( + service.locate(Link(href: "/href1")), + Locator(href: "/href1", type: "text/html", locations: Locator.Locations(progression: 0.0)) + ) + + XCTAssertEqual( + service.locate(Link(href: "/href2")), + Locator(href: "/href2", type: "text/html", locations: Locator.Locations(progression: 0.0)) + ) + + XCTAssertEqual( + service.locate(Link(href: "/href3")), + Locator(href: "/href3", type: "text/html", locations: Locator.Locations(progression: 0.0)) + ) + } + + func testFromLinkWithFragment() { + let service = makeService(readingOrder: [ + Link(href: "/href", type: "text/html", title: "Resource") + ]) + + XCTAssertEqual( + service.locate(Link(href: "/href#page=42", type: "text/xml", title: "My link")), + Locator(href: "/href", type: "text/html", title: "Resource", locations: Locator.Locations(fragments: ["page=42"])) + ) + } + + func testTitleFallbackFromLink() { + let service = makeService(readingOrder: [ + Link(href: "/href", type: "text/html") + ]) + + XCTAssertEqual( + service.locate(Link(href: "/href", title: "My link")), + Locator(href: "/href", type: "text/html", title: "My link", locations: Locator.Locations(progression: 0.0)) + ) + } + + func testFromLinkNotFound() { + let service = makeService(readingOrder: [ + Link(href: "/href", type: "text/html") + ]) + + XCTAssertNil(service.locate(Link(href: "notfound"))) + } + + func makeService( + links: [Link] = [], + readingOrder: [Link] = [], + resources: [Link] = [], + positions: [[Locator]] = [] + ) -> DefaultLocatorService { DefaultLocatorService(publication: _Strong(Publication( - manifest: Manifest(metadata: Metadata(title: ""), readingOrder: readingOrder), + manifest: Manifest( + metadata: Metadata(title: ""), + links: links, + readingOrder: readingOrder, + resources: resources + ), servicesBuilder: PublicationServicesBuilder( positions: InMemoryPositionsService.makeFactory(positionsByReadingOrder: positions) ) From 935a51cea7a63e055e97af02b9e37e39f17b014c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 27 Apr 2022 18:36:04 +0200 Subject: [PATCH 03/46] Update changelog --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c94d38914..21f3e7e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,15 @@ All notable changes to this project will be documented in this file. Take a look **Warning:** Features marked as *alpha* may change or be removed in a future release without notice. Use with caution. - +## [Unreleased] + +### Shared + +#### Deprecated + +* `Locator(link: Link)` is deprecated as it may create an incorrect `Locator` if the link `type` is missing. + * Use `publication.locate(Link)` instead. + ## [2.3.0] From d6d3deab4d8c8e19f454f3b1ed3144ed5cd00483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 27 Apr 2022 18:50:19 +0200 Subject: [PATCH 04/46] Update Carthage project --- .gitignore | 2 +- Support/Carthage/Readium.xcodeproj/project.pbxproj | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ba719ac08..0c5fcb4e6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ Package.resolved # Carthage -Carthage/ +./Carthage/ Cartfile.resolved # Xcode diff --git a/Support/Carthage/Readium.xcodeproj/project.pbxproj b/Support/Carthage/Readium.xcodeproj/project.pbxproj index 2458f598c..8b2068588 100644 --- a/Support/Carthage/Readium.xcodeproj/project.pbxproj +++ b/Support/Carthage/Readium.xcodeproj/project.pbxproj @@ -274,6 +274,7 @@ E8CB4E5729E7000FC55FC937 /* CoreServices.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 342D5C0FEE79A2ABEE24A43E /* CoreServices.framework */; }; E9AADF25494C968A44979B66 /* UInt64.swift in Sources */ = {isa = PBXBuildFile; fileRef = 57338C29681D4872D425AB81 /* UInt64.swift */; }; E9EE047B89D0084523F7C888 /* Feed.swift in Sources */ = {isa = PBXBuildFile; fileRef = C5E7CEDF6EA681FE8119791B /* Feed.swift */; }; + EA8C7F894E3BE8D6D954DC47 /* InMemoryPositionsService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 505BF8A630F7C7B96754E333 /* InMemoryPositionsService.swift */; }; EB4D11D2D1A0C64FF0E982C3 /* Optional.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC925E451D875E5F74748EDC /* Optional.swift */; }; EDCA3449EA5683B37D82FEBE /* PDFKit.swift in Sources */ = {isa = PBXBuildFile; fileRef = ABAF1D0444B94E2CDD80087D /* PDFKit.swift */; }; EE951A131E38E316BF7A1129 /* LCPDialogViewController.xib in Resources */ = {isa = PBXBuildFile; fileRef = ED5C6546C24E5E619E4CC9D1 /* LCPDialogViewController.xib */; }; @@ -412,6 +413,7 @@ 4BB5D42EEF0083D833E2A572 /* Publication+OPDS.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Publication+OPDS.swift"; sourceTree = ""; }; 4E564AE6D5137499C81FEBE2 /* TargetAction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TargetAction.swift; sourceTree = ""; }; 500E55D9CA753D6D6AA76D10 /* EPUBLicenseContainer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EPUBLicenseContainer.swift; sourceTree = ""; }; + 505BF8A630F7C7B96754E333 /* InMemoryPositionsService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InMemoryPositionsService.swift; sourceTree = ""; }; 508E0CD4F9F02CC851E6D1E1 /* Publication+EPUB.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Publication+EPUB.swift"; sourceTree = ""; }; 54699BC0E00F327E67908F6A /* Encryption.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Encryption.swift; sourceTree = ""; }; 55BC4119B8937D17ED80B1AB /* ControlFlow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ControlFlow.swift; sourceTree = ""; }; @@ -982,6 +984,7 @@ 5BC52D8F4F854FDA56D10A8E /* Positions */ = { isa = PBXGroup; children = ( + 505BF8A630F7C7B96754E333 /* InMemoryPositionsService.swift */, 01CCE64AE9824DCF6D6413BC /* PerResourcePositionsService.swift */, BC45956B8991A9488F957B06 /* PositionsService.swift */, ); @@ -1882,6 +1885,7 @@ 39FC65D3797EF5069A04F34B /* HTTPFetcher.swift in Sources */, 2B57BE89EFAE517F79A17667 /* HTTPProblemDetails.swift in Sources */, FD13DEAC62A3ED6714841B7A /* HTTPRequest.swift in Sources */, + EA8C7F894E3BE8D6D954DC47 /* InMemoryPositionsService.swift in Sources */, 9C6B7AFB6FB0635EF5B7B71C /* JSON.swift in Sources */, 69150D0B00F5665C3DA0000B /* LazyResource.swift in Sources */, 5C9617AE1B5678A95ABFF1AA /* Link.swift in Sources */, From 0cb62a286b96eb04b7e2a605619d5d2dd0e1f5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 28 Apr 2022 11:36:25 +0200 Subject: [PATCH 05/46] Add the `PublicationContentIterator` --- .../Services/Content/ContentIterator.swift | 198 ++++++++++++++++++ .../Services/Search/SearchService.swift | 2 +- Sources/Shared/Toolkit/Cancellable.swift | 12 ++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 Sources/Shared/Publication/Services/Content/ContentIterator.swift diff --git a/Sources/Shared/Publication/Services/Content/ContentIterator.swift b/Sources/Shared/Publication/Services/Content/ContentIterator.swift new file mode 100644 index 000000000..b0e513719 --- /dev/null +++ b/Sources/Shared/Publication/Services/Content/ContentIterator.swift @@ -0,0 +1,198 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +public struct Content: Equatable { + public let locator: Locator + public let data: Data + + public var extras: [String: Any] { + get { extrasJSON.json } + set { extrasJSON = JSONDictionary(newValue) ?? JSONDictionary() } + } + // Trick to keep the struct equatable despite [String: Any] + private var extrasJSON: JSONDictionary + + public init(locator: Locator, data: Data, extras: [String: Any] = [:]) { + self.locator = locator + self.data = data + self.extrasJSON = JSONDictionary(extras) ?? JSONDictionary() + } + + public enum Data: Equatable { + case audio(target: Link) + case image(target: Link, description: String?) + case text(spans: TextSpan, style: TextStyle) + } + + public enum TextStyle: Equatable { + case heading(level: Int) + case body + case callout + case caption + case footnote + case quote + case listItem + } + + public struct TextSpan: Equatable { + let locator: Locator + let language: String? + let text: String + } +} + +public protocol ContentIterator: AnyObject { + @discardableResult + func previous(completion: @escaping (Result) -> Void) -> Cancellable + + @discardableResult + func next(completion: @escaping (Result) -> Void) -> Cancellable + + func close() +} + + +/// Creates a `ContentIterator` instance for the given `resource`. +/// +/// - Returns: nil if the resource format is not supported. +typealias ResourceContentIteratorFactory = + (_ resource: Resource, _ locator: Locator) -> ContentIterator? + +class PublicationContentIterator: ContentIterator { + + private let publication: Publication + private var startLocator: Locator? + private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] + private let startIndex: Int + private var currentIndex: Int + private var currentIterator: ContentIterator? + + private let queue = DispatchQueue(label: "org.readium.shared.PublicationContentIterator") + + init(publication: Publication, start: Locator?, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { + self.publication = publication + self.startLocator = start + self.resourceContentIteratorFactories = resourceContentIteratorFactories + + startIndex = { + guard + let start = start, + let index = publication.readingOrder.firstIndex(withHREF: start.href) + else { + return 0 + } + return index + }() + + currentIndex = startIndex + } + + func previous(completion: @escaping (Result) -> Void) -> Cancellable { + return CancellableObject() + } + + func next(completion: @escaping (Result) -> Void) -> Cancellable { + let cancellable = MediatorCancellable() + + func finish(_ result: Result) { + guard !cancellable.isCancelled else { + return + } + DispatchQueue.main.async { + completion(result) + } + } + + queue.async { [self] in + guard let iterator = iterator(by: +1) else { + finish(.success(nil)) + return + } + guard !cancellable.isCancelled else { + return + } + + iterator + .next { result in + switch result { + case .success(let content): + if let content = content { + finish(.success(content)) + } else { + next(completion: completion) + .mediated(by: cancellable) + } + case .failure(let error): + finish(.failure(error)) + } + } + .mediated(by: cancellable) + } + + return cancellable + } + + func iterator(by delta: Int) -> ContentIterator? { + if let iter = currentIterator { + return iter + } + guard let (newIndex, newIterator) = loadIterator(from: currentIndex, by: delta) else { + return nil + } + currentIndex = newIndex + currentIterator = newIterator + return newIterator + } + + func loadIterator(from index: Int, by delta: Int) -> (index: Int, ContentIterator)? { + let i = index + delta + guard publication.readingOrder.indices.contains(i) else { + return nil + } + guard let iterator = loadIterator(at: i) else { + return loadIterator(from: i, by: delta) + } + return (i, iterator) + } + + func loadIterator(at index: Int) -> ContentIterator? { + let link = publication.readingOrder[index] + guard var locator = publication.locate(link) else { + return nil + } + + if let start = startLocator.pop() { + locator = locator.copy( + locations: { $0 = start.locations }, + text: { $0 = start.text } + ) + } + + let resource = publication.get(link) + for factory in resourceContentIteratorFactories { + if let iterator = factory(resource, locator) { + return iterator + } + } + + return nil + } + + func close() { + currentIterator?.close() + currentIterator = nil + } +} + +public extension Optional { + mutating func pop() -> Wrapped? { + let res = self + self = nil + return res + } +} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Search/SearchService.swift b/Sources/Shared/Publication/Services/Search/SearchService.swift index 4966d6448..603faeb8f 100644 --- a/Sources/Shared/Publication/Services/Search/SearchService.swift +++ b/Sources/Shared/Publication/Services/Search/SearchService.swift @@ -26,7 +26,7 @@ public protocol _SearchService: PublicationService { } /// Iterates through search results. -public protocol SearchIterator { +public protocol SearchIterator: AnyObject { /// Number of matches for this search, if known. /// diff --git a/Sources/Shared/Toolkit/Cancellable.swift b/Sources/Shared/Toolkit/Cancellable.swift index dd79ac203..bf3bb119c 100644 --- a/Sources/Shared/Toolkit/Cancellable.swift +++ b/Sources/Shared/Toolkit/Cancellable.swift @@ -59,3 +59,15 @@ public final class MediatorCancellable: Cancellable { cancellable = nil } } + +public extension Cancellable { + /// Convenience to mediate a cancellable in a call chain. + /// + /// ``` + /// apiReturningACancellable() + /// .mediate(by: mediator) + /// ``` + func mediated(by mediator: MediatorCancellable) { + mediator.mediate(self) + } +} \ No newline at end of file From 08e88ece2bdfd6da8421259796aec348bb5192d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 28 Apr 2022 12:14:28 +0200 Subject: [PATCH 06/46] Simplify the ContentIterator by making it synchronous --- .../Services/Content/ContentIterator.swift | 147 +----------------- .../Content/PublicationContentIterator.swift | 106 +++++++++++++ .../Shared/Toolkit/Extensions/Optional.swift | 8 +- 3 files changed, 115 insertions(+), 146 deletions(-) create mode 100644 Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift diff --git a/Sources/Shared/Publication/Services/Content/ContentIterator.swift b/Sources/Shared/Publication/Services/Content/ContentIterator.swift index b0e513719..41d5520f7 100644 --- a/Sources/Shared/Publication/Services/Content/ContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/ContentIterator.swift @@ -47,152 +47,9 @@ public struct Content: Equatable { } public protocol ContentIterator: AnyObject { - @discardableResult - func previous(completion: @escaping (Result) -> Void) -> Cancellable - - @discardableResult - func next(completion: @escaping (Result) -> Void) -> Cancellable func close() + func previous() throws -> Content? + func next() throws -> Content? } - -/// Creates a `ContentIterator` instance for the given `resource`. -/// -/// - Returns: nil if the resource format is not supported. -typealias ResourceContentIteratorFactory = - (_ resource: Resource, _ locator: Locator) -> ContentIterator? - -class PublicationContentIterator: ContentIterator { - - private let publication: Publication - private var startLocator: Locator? - private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] - private let startIndex: Int - private var currentIndex: Int - private var currentIterator: ContentIterator? - - private let queue = DispatchQueue(label: "org.readium.shared.PublicationContentIterator") - - init(publication: Publication, start: Locator?, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { - self.publication = publication - self.startLocator = start - self.resourceContentIteratorFactories = resourceContentIteratorFactories - - startIndex = { - guard - let start = start, - let index = publication.readingOrder.firstIndex(withHREF: start.href) - else { - return 0 - } - return index - }() - - currentIndex = startIndex - } - - func previous(completion: @escaping (Result) -> Void) -> Cancellable { - return CancellableObject() - } - - func next(completion: @escaping (Result) -> Void) -> Cancellable { - let cancellable = MediatorCancellable() - - func finish(_ result: Result) { - guard !cancellable.isCancelled else { - return - } - DispatchQueue.main.async { - completion(result) - } - } - - queue.async { [self] in - guard let iterator = iterator(by: +1) else { - finish(.success(nil)) - return - } - guard !cancellable.isCancelled else { - return - } - - iterator - .next { result in - switch result { - case .success(let content): - if let content = content { - finish(.success(content)) - } else { - next(completion: completion) - .mediated(by: cancellable) - } - case .failure(let error): - finish(.failure(error)) - } - } - .mediated(by: cancellable) - } - - return cancellable - } - - func iterator(by delta: Int) -> ContentIterator? { - if let iter = currentIterator { - return iter - } - guard let (newIndex, newIterator) = loadIterator(from: currentIndex, by: delta) else { - return nil - } - currentIndex = newIndex - currentIterator = newIterator - return newIterator - } - - func loadIterator(from index: Int, by delta: Int) -> (index: Int, ContentIterator)? { - let i = index + delta - guard publication.readingOrder.indices.contains(i) else { - return nil - } - guard let iterator = loadIterator(at: i) else { - return loadIterator(from: i, by: delta) - } - return (i, iterator) - } - - func loadIterator(at index: Int) -> ContentIterator? { - let link = publication.readingOrder[index] - guard var locator = publication.locate(link) else { - return nil - } - - if let start = startLocator.pop() { - locator = locator.copy( - locations: { $0 = start.locations }, - text: { $0 = start.text } - ) - } - - let resource = publication.get(link) - for factory in resourceContentIteratorFactories { - if let iterator = factory(resource, locator) { - return iterator - } - } - - return nil - } - - func close() { - currentIterator?.close() - currentIterator = nil - } -} - -public extension Optional { - mutating func pop() -> Wrapped? { - let res = self - self = nil - return res - } -} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift new file mode 100644 index 000000000..620be9f34 --- /dev/null +++ b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift @@ -0,0 +1,106 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// Creates a `ContentIterator` instance for the given `resource`. +/// +/// - Returns: nil if the resource format is not supported. +public typealias ResourceContentIteratorFactory = + (_ resource: Resource, _ locator: Locator) -> ContentIterator? + +public class PublicationContentIterator: ContentIterator { + + private let publication: Publication + private var startLocator: Locator? + private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] + private let startIndex: Int + private var currentIndex: Int + private var currentIterator: ContentIterator? + + public init(publication: Publication, start: Locator?, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { + self.publication = publication + self.startLocator = start + self.resourceContentIteratorFactories = resourceContentIteratorFactories + + startIndex = { + guard + let start = start, + let index = publication.readingOrder.firstIndex(withHREF: start.href) + else { + return 0 + } + return index + }() + + currentIndex = startIndex + } + + public func close() { + currentIterator?.close() + currentIterator = nil + } + + public func previous() throws -> Content? { + nil + } + + public func next() throws -> Content? { + guard let iterator = iterator(by: +1) else { + return nil + } + guard let content = try iterator.next() else { + return try next() + } + return content + } + + private func iterator(by delta: Int) -> ContentIterator? { + if let iter = currentIterator { + return iter + } + guard let (newIndex, newIterator) = loadIterator(from: currentIndex, by: delta) else { + return nil + } + currentIndex = newIndex + currentIterator = newIterator + return newIterator + } + + private func loadIterator(from index: Int, by delta: Int) -> (index: Int, ContentIterator)? { + let i = index + delta + guard publication.readingOrder.indices.contains(i) else { + return nil + } + guard let iterator = loadIterator(at: i) else { + return loadIterator(from: i, by: delta) + } + return (i, iterator) + } + + private func loadIterator(at index: Int) -> ContentIterator? { + let link = publication.readingOrder[index] + guard var locator = publication.locate(link) else { + return nil + } + + if let start = startLocator.pop() { + locator = locator.copy( + locations: { $0 = start.locations }, + text: { $0 = start.text } + ) + } + + let resource = publication.get(link) + for factory in resourceContentIteratorFactories { + if let iterator = factory(resource, locator) { + return iterator + } + } + + return nil + } +} diff --git a/Sources/Shared/Toolkit/Extensions/Optional.swift b/Sources/Shared/Toolkit/Extensions/Optional.swift index 3c69afddd..69dc6193a 100644 --- a/Sources/Shared/Toolkit/Extensions/Optional.swift +++ b/Sources/Shared/Toolkit/Extensions/Optional.swift @@ -28,5 +28,11 @@ extension Optional { } return value } - + + /// Returns the wrapped value and modify the variable to be nil. + mutating func pop() -> Wrapped? { + let res = self + self = nil + return res + } } From 6ed6a7ef2d2d11dcda25d5d73d76f6cc019f467d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 28 Apr 2022 18:39:16 +0200 Subject: [PATCH 07/46] Add the `HTMLResourceContentIterator` and `ContentIterationService` --- .../Shared/Fetcher/Resource/Resource.swift | 7 +- .../ContentProtectionService.swift | 17 +- .../Content/ContentIterationService.swift | 64 +++++ .../Services/Content/ContentIterator.swift | 6 +- .../Content/HTMLResourceContentIterator.swift | 266 ++++++++++++++++++ .../Content/PublicationContentIterator.swift | 20 +- .../Services/PublicationServicesBuilder.swift | 13 +- .../Shared/Toolkit/Extensions/Result.swift | 19 ++ Sources/Streamer/Parser/EPUB/EPUBParser.swift | 5 + .../Sources/Reader/Common/ColorScheme.swift | 3 +- 10 files changed, 383 insertions(+), 37 deletions(-) create mode 100644 Sources/Shared/Publication/Services/Content/ContentIterationService.swift create mode 100644 Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift diff --git a/Sources/Shared/Fetcher/Resource/Resource.swift b/Sources/Shared/Fetcher/Resource/Resource.swift index d56ef14f5..ef0527b0c 100644 --- a/Sources/Shared/Fetcher/Resource/Resource.swift +++ b/Sources/Shared/Fetcher/Resource/Resource.swift @@ -235,7 +235,7 @@ public extension Result where Failure == ResourceError { /// /// If the `transform` throws an `Error`, it is wrapped in a failure with `Resource.Error.Other`. func tryMap(_ transform: (Success) throws -> NewSuccess) -> ResourceResult { - return flatMap { + flatMap { do { return .success(try transform($0)) } catch { @@ -245,7 +245,6 @@ public extension Result where Failure == ResourceError { } func tryFlatMap(_ transform: (Success) throws -> ResourceResult) -> ResourceResult { - return tryMap(transform).flatMap { $0 } + tryMap(transform).flatMap { $0 } } - -} +} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content Protection/ContentProtectionService.swift b/Sources/Shared/Publication/Services/Content Protection/ContentProtectionService.swift index a983d365c..37aa4dc38 100644 --- a/Sources/Shared/Publication/Services/Content Protection/ContentProtectionService.swift +++ b/Sources/Shared/Publication/Services/Content Protection/ContentProtectionService.swift @@ -1,12 +1,7 @@ // -// ContentProtectionService.swift -// r2-shared-swift -// -// Created by Mickaël Menu on 16/07/2020. -// -// Copyright 2020 Readium Foundation. All rights reserved. -// Use of this source code is governed by a BSD-style license which is detailed -// in the LICENSE file present in the project repository where this source code is maintained. +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. // import Foundation @@ -40,17 +35,13 @@ public protocol ContentProtectionService: PublicationService { /// /// It could be used in a sentence such as "Protected by {name}". var name: LocalizedString? { get } - } public extension ContentProtectionService { var credentials: String? { nil } - var rights: UserRights { UnrestrictedUserRights() } - var name: LocalizedString? { nil } - } @@ -108,7 +99,6 @@ public extension Publication { private var contentProtectionService: ContentProtectionService? { findService(ContentProtectionService.self) } - } @@ -123,5 +113,4 @@ public extension PublicationServicesBuilder { remove(ContentProtectionService.self) } } - } diff --git a/Sources/Shared/Publication/Services/Content/ContentIterationService.swift b/Sources/Shared/Publication/Services/Content/ContentIterationService.swift new file mode 100644 index 000000000..1c89dc392 --- /dev/null +++ b/Sources/Shared/Publication/Services/Content/ContentIterationService.swift @@ -0,0 +1,64 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +public typealias ContentIterationServiceFactory = (PublicationServiceContext) -> ContentIterationService? + +public protocol ContentIterationService: PublicationService { + func iterator(from start: Locator?) -> ContentIterator? +} + +public extension Publication { + func contentIterator(from start: Locator?) -> ContentIterator? { + findService(ContentIterationService.self)? + .iterator(from: start) + } +} + +public extension PublicationServicesBuilder { + mutating func setContentIterationServiceFactory(_ factory: ContentIterationServiceFactory?) { + if let factory = factory { + set(ContentIterationService.self, factory) + } else { + remove(ContentIterationService.self) + } + } +} + +public class DefaultContentIterationService: ContentIterationService { + + public static func makeFactory(resourceContentIteratorFactories: [ResourceContentIteratorFactory]) -> (PublicationServiceContext) -> DefaultContentIterationService? { + { context in + DefaultContentIterationService( + publication: context.publication, + resourceContentIteratorFactories: resourceContentIteratorFactories + ) + } + } + + private let publication: Weak + private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] + + public init( + publication: Weak, + resourceContentIteratorFactories: [ResourceContentIteratorFactory] + ) { + self.publication = publication + self.resourceContentIteratorFactories = resourceContentIteratorFactories + } + + public func iterator(from start: Locator?) -> ContentIterator? { + guard let publication = publication() else { + return nil + } + return PublicationContentIterator( + publication: publication, + start: start, + resourceContentIteratorFactories: resourceContentIteratorFactories + ) + } +} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/ContentIterator.swift b/Sources/Shared/Publication/Services/Content/ContentIterator.swift index 41d5520f7..b049233df 100644 --- a/Sources/Shared/Publication/Services/Content/ContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/ContentIterator.swift @@ -26,7 +26,7 @@ public struct Content: Equatable { public enum Data: Equatable { case audio(target: Link) case image(target: Link, description: String?) - case text(spans: TextSpan, style: TextStyle) + case text(spans: [TextSpan], style: TextStyle) } public enum TextStyle: Equatable { @@ -47,9 +47,7 @@ public struct Content: Equatable { } public protocol ContentIterator: AnyObject { - func close() func previous() throws -> Content? func next() throws -> Content? -} - +} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift b/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift new file mode 100644 index 000000000..c3b58c7e6 --- /dev/null +++ b/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift @@ -0,0 +1,266 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation +import SwiftSoup + +public class HTMLResourceContentIterator : ContentIterator { + + // FIXME: Custom skipped elements + public static func makeFactory() -> ResourceContentIteratorFactory { + { resource, locator in + HTMLResourceContentIterator(resource: resource, locator: locator) + } + } + + private var content: Result<[Content], Error> + private var currentIndex: Int? + private var startingIndex: Int + + public init(resource: Resource, locator: Locator) { + let result = resource + .readAsString() + .eraseToAnyError() + .tryMap { try SwiftSoup.parse($0) } + .tryMap { document -> ContentParser in + let parser = ContentParser( + baseLocator: locator, + startElement: try locator.locations.cssSelector + .flatMap { + // The JS third-party library used to generate the CSS Selector sometimes adds + // :root >, which doesn't work with JSoup. + try document.select($0.removingPrefix(":root > ")).first() + } + ) + try document.traverse(parser) + return parser + } + + content = result.map { $0.content } + startingIndex = result.map { $0.startIndex }.get(or: 0) + } + + public func close() {} + + public func previous() throws -> Content? { + try next(by: -1) + } + + public func next() throws -> Content? { + try next(by: +1) + } + + private func next(by delta: Int) throws -> Content? { + let content = try content.get() + let index = index(by: delta) + guard content.indices.contains(index) else { + return nil + } + currentIndex = index + return content[index] + } + + private func index(by delta: Int) -> Int { + if let i = currentIndex { + return i + delta + } else { + return startingIndex + } + } + + private class ContentParser: NodeVisitor { + private let baseLocator: Locator + private let startElement: Element? + + private(set) var content: [Content] = [] + private(set) var startIndex = 0 + private var currentElement: Element? + private var spansAcc: [Content.TextSpan] = [] + private var textAcc = StringBuilder() + private var wholeRawTextAcc: String = "" + private var elementRawTextAcc: String = "" + private var rawTextAcc: String = "" + private var currentLanguage: String? + private var currentCSSSelector: String? + private var ignoredNode: Node? + + init(baseLocator: Locator, startElement: Element?) { + self.baseLocator = baseLocator + self.startElement = startElement + } + + public func head(_ node: Node, _ depth: Int) throws { + guard ignoredNode == nil else { + return + } + guard !node.isHidden else { + ignoredNode = node + return + } + + if let elem = node as? Element { + currentElement = elem + + let cssSelector = try elem.cssSelector() + let tag = elem.tagNameNormal() + + if tag == "br" { + flushText() + } else if tag == "img" { + flushText() + + if let href = try elem.attr("src") + .takeUnlessEmpty() + .map({ HREF($0, relativeTo: baseLocator.href).string }) { + content.append(Content( + locator: baseLocator.copy( + locations: { + $0 = Locator.Locations( + otherLocations: ["cssSelector": cssSelector] + ) + } + ), + data: .image( + target: Link(href: href), + description: try elem.attr("alt").takeUnlessEmpty() + ) + )) + } + + } else if elem.isBlock() { + spansAcc.removeAll() + textAcc.clear() + rawTextAcc = "" + currentCSSSelector = cssSelector + } + } + } + + func tail(_ node: Node, _ depth: Int) throws { + if ignoredNode == node { + ignoredNode = nil + } + + if let node = node as? TextNode { + let language = try node.language() + if (currentLanguage != language) { + flushSpan() + currentLanguage = language + } + + rawTextAcc += try Parser.unescapeEntities(node.getWholeText(), false) + try appendNormalisedText(of: node) + + } else if let node = node as? Element { + if node.isBlock() { + flushText() + } + } + } + + private func appendNormalisedText(of textNode: TextNode) throws { + let text = try Parser.unescapeEntities(textNode.getWholeText(), false) + return StringUtil.appendNormalisedWhitespace(textAcc, string: text, stripLeading: lastCharIsWhitespace()) + } + + private func lastCharIsWhitespace() -> Bool { + textAcc.toString().last?.isWhitespace ?? false + } + + private func flushText() { + flushSpan() + guard !spansAcc.isEmpty else { + return + } + + if startElement != nil && currentElement == startElement { + startIndex = content.count + } + content.append(Content( + locator: baseLocator.copy( + locations: { [self] in + $0 = Locator.Locations( + otherLocations: [ + "cssSelector": currentCSSSelector as Any + ] + ) + }, + text: { [self] in + $0 = Locator.Text( + highlight: elementRawTextAcc + ) + } + ), + data: .text(spans: spansAcc, style: .body) + )) + elementRawTextAcc = "" + spansAcc.removeAll() + } + + private func flushSpan() { + var text = textAcc.toString() + let trimmedText = text.trimmingCharacters(in: .whitespacesAndNewlines) + + if !text.isEmpty { + if spansAcc.isEmpty { + let whitespaceSuffix = text.last + .takeIf { $0.isWhitespace || $0.isNewline } + .map { String($0) } + ?? "" + + text = trimmedText + whitespaceSuffix + } + + spansAcc.append(Content.TextSpan( + locator: baseLocator.copy( + locations: { [self] in + $0 = Locator.Locations( + otherLocations: [ + "cssSelector": currentCSSSelector as Any + ] + ) + }, + text: { [self] in + $0 = Locator.Text( + before: String(wholeRawTextAcc.suffix(50)), + highlight: rawTextAcc // FIXME: custom length + ) + } + ), + language: currentLanguage, + text: text + )) + } + + wholeRawTextAcc += rawTextAcc + elementRawTextAcc += rawTextAcc + rawTextAcc = "" + textAcc.clear() + } + } +} + +private extension Node { + // FIXME: Setup ignore conditions + var isHidden: Bool { false } + + func language() throws -> String? { + try attr("xml:lang").takeUnlessEmpty() + ?? attr("lang").takeUnlessEmpty() + ?? parent()?.language() + } + + func parentElement() -> Element? { + (parent() as? Element) + ?? parent()?.parentElement() + } +} + +private extension String { + func takeUnlessEmpty() -> String? { + isEmpty ? nil : self + } +} diff --git a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift index 620be9f34..01c5d558b 100644 --- a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift @@ -12,13 +12,13 @@ import Foundation public typealias ResourceContentIteratorFactory = (_ resource: Resource, _ locator: Locator) -> ContentIterator? -public class PublicationContentIterator: ContentIterator { +public class PublicationContentIterator: ContentIterator, Loggable { private let publication: Publication private var startLocator: Locator? private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] - private let startIndex: Int - private var currentIndex: Int + private var startIndex: Int? + private var currentIndex: Int = 0 private var currentIterator: ContentIterator? public init(publication: Publication, start: Locator?, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { @@ -30,13 +30,11 @@ public class PublicationContentIterator: ContentIterator { guard let start = start, let index = publication.readingOrder.firstIndex(withHREF: start.href) - else { + else { return 0 } return index }() - - currentIndex = startIndex } public func close() { @@ -53,6 +51,7 @@ public class PublicationContentIterator: ContentIterator { return nil } guard let content = try iterator.next() else { + currentIterator = nil return try next() } return content @@ -62,6 +61,15 @@ public class PublicationContentIterator: ContentIterator { if let iter = currentIterator { return iter } + + // For the first requested iterator, we don't want to move by the given delta. + var delta = delta + if let start = startIndex { + startIndex = nil + currentIndex = start + delta = 0 + } + guard let (newIndex, newIterator) = loadIterator(from: currentIndex, by: delta) else { return nil } diff --git a/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift b/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift index 2004e4533..117f4ba7b 100644 --- a/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift +++ b/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift @@ -1,12 +1,7 @@ // -// PublicationServicesBuilder.swift -// r2-shared-swift -// -// Created by Mickaël Menu on 30/05/2020. -// -// Copyright 2020 Readium Foundation. All rights reserved. -// Use of this source code is governed by a BSD-style license which is detailed -// in the LICENSE file present in the project repository where this source code is maintained. +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. // import Foundation @@ -19,6 +14,7 @@ public struct PublicationServicesBuilder { private var factories: [String: PublicationServiceFactory] = [:] public init( + contentIteration: ContentIterationServiceFactory? = nil, contentProtection: ContentProtectionServiceFactory? = nil, cover: CoverServiceFactory? = nil, locator: LocatorServiceFactory? = { DefaultLocatorService(publication: $0.publication) }, @@ -26,6 +22,7 @@ public struct PublicationServicesBuilder { search: SearchServiceFactory? = nil, setup: (inout PublicationServicesBuilder) -> Void = { _ in } ) { + setContentIterationServiceFactory(contentIteration) setContentProtectionServiceFactory(contentProtection) setCoverServiceFactory(cover) setLocatorServiceFactory(locator) diff --git a/Sources/Shared/Toolkit/Extensions/Result.swift b/Sources/Shared/Toolkit/Extensions/Result.swift index e479b443b..dd6f41ee3 100644 --- a/Sources/Shared/Toolkit/Extensions/Result.swift +++ b/Sources/Shared/Toolkit/Extensions/Result.swift @@ -17,6 +17,10 @@ extension Result { return try? get() } + func get(or def: Success) -> Success { + (try? get()) ?? def + } + func `catch`(_ recover: (Failure) -> Self) -> Self { if case .failure(let error) = self { return recover(error) @@ -24,4 +28,19 @@ extension Result { return self } + func eraseToAnyError() -> Result { + mapError { $0 as Error } + } +} + +extension Result where Failure == Error { + func tryMap(_ transform:(Success) throws -> T) -> Result { + flatMap { + do { + return .success(try transform($0)) + } catch { + return .failure(error) + } + } + } } diff --git a/Sources/Streamer/Parser/EPUB/EPUBParser.swift b/Sources/Streamer/Parser/EPUB/EPUBParser.swift index b76c6d3da..032521452 100644 --- a/Sources/Streamer/Parser/EPUB/EPUBParser.swift +++ b/Sources/Streamer/Parser/EPUB/EPUBParser.swift @@ -85,6 +85,11 @@ final public class EPUBParser: PublicationParser { EPUBHTMLInjector(metadata: components.metadata, userProperties: userProperties).inject(resource:) ]), servicesBuilder: .init( + contentIteration: DefaultContentIterationService.makeFactory( + resourceContentIteratorFactories: [ + HTMLResourceContentIterator.makeFactory() + ] + ), positions: EPUBPositionsService.makeFactory(reflowableStrategy: reflowablePositionsStrategy), search: _StringSearchService.makeFactory() ), diff --git a/TestApp/Sources/Reader/Common/ColorScheme.swift b/TestApp/Sources/Reader/Common/ColorScheme.swift index 23cdecd61..81c58d74f 100644 --- a/TestApp/Sources/Reader/Common/ColorScheme.swift +++ b/TestApp/Sources/Reader/Common/ColorScheme.swift @@ -22,7 +22,8 @@ class ColorScheme { struct ColorModifier: ViewModifier { let colorScheme: ColorScheme - func body(content: Content) -> some View { + + func body(content: Self.Content) -> some View { content .foregroundColor(colorScheme.textColor) .background(colorScheme.mainColor) From c5de09b36ce760fb3513c1b0793f96c90938b13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 2 May 2022 11:46:50 +0200 Subject: [PATCH 08/46] Add findFirstVisibleElement() JS API --- .../Assets/Static/scripts/readium-fixed.js | 183 ++++++++++++++++-- .../Static/scripts/readium-reflowable.js | 183 ++++++++++++++++-- Sources/Navigator/EPUB/Scripts/package.json | 5 +- Sources/Navigator/EPUB/Scripts/src/dom.js | 98 ++++++++++ .../Navigator/EPUB/Scripts/src/index-fixed.js | 2 + .../EPUB/Scripts/src/index-reflowable.js | 2 + Sources/Navigator/EPUB/Scripts/src/index.js | 4 + Sources/Navigator/EPUB/Scripts/src/utils.js | 17 +- Sources/Navigator/EPUB/Scripts/yarn.lock | 25 +++ 9 files changed, 484 insertions(+), 35 deletions(-) create mode 100644 Sources/Navigator/EPUB/Scripts/src/dom.js diff --git a/Sources/Navigator/EPUB/Assets/Static/scripts/readium-fixed.js b/Sources/Navigator/EPUB/Assets/Static/scripts/readium-fixed.js index c15ba4e05..90d0a3672 100644 --- a/Sources/Navigator/EPUB/Assets/Static/scripts/readium-fixed.js +++ b/Sources/Navigator/EPUB/Assets/Static/scripts/readium-fixed.js @@ -1612,6 +1612,132 @@ window.addEventListener("load", function () { /***/ }), +/***/ "./src/dom.js": +/*!********************!*\ + !*** ./src/dom.js ***! + \********************/ +/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { + +"use strict"; +__webpack_require__.r(__webpack_exports__); +/* harmony export */ __webpack_require__.d(__webpack_exports__, { +/* harmony export */ "findFirstVisibleLocator": () => (/* binding */ findFirstVisibleLocator) +/* harmony export */ }); +/* harmony import */ var _utils__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./utils */ "./src/utils.js"); +/* harmony import */ var css_selector_generator__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! css-selector-generator */ "./node_modules/css-selector-generator/build/index.js"); +/* harmony import */ var css_selector_generator__WEBPACK_IMPORTED_MODULE_1___default = /*#__PURE__*/__webpack_require__.n(css_selector_generator__WEBPACK_IMPORTED_MODULE_1__); +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + + +function findFirstVisibleLocator() { + var element = findElement(document.body); + + if (!element) { + return undefined; + } + + return { + href: "#", + type: "application/xhtml+xml", + locations: { + cssSelector: (0,css_selector_generator__WEBPACK_IMPORTED_MODULE_1__.getCssSelector)(element) + }, + text: { + highlight: element.textContent + } + }; +} + +function findElement(rootElement) { + var foundElement = undefined; + + for (var i = rootElement.children.length - 1; i >= 0; i--) { + var child = rootElement.children[i]; + var position = elementRelativePosition(child, undefined); + + if (position == 0) { + if (!shouldIgnoreElement(child)) { + foundElement = child; + } + } else if (position < 0) { + if (!foundElement) { + foundElement = child; + } + + break; + } + } + + if (foundElement) { + return findElement(foundElement); + } + + return rootElement; +} // See computeVisibility_() in r2-navigator-js + + +function elementRelativePosition(element, domRect +/* nullable */ +) { + if (readium.isFixedLayout) return true; + + if (element === document.body || element === document.documentElement) { + return -1; + } + + if (!document || !document.documentElement || !document.body) { + return 1; + } + + var rect = domRect || element.getBoundingClientRect(); + + if ((0,_utils__WEBPACK_IMPORTED_MODULE_0__.isScrollModeEnabled)()) { + return rect.top >= 0 && rect.top <= document.documentElement.clientHeight; + } else { + var pageWidth = window.innerWidth; + + if (rect.left >= pageWidth) { + return 1; + } else if (rect.left >= 0) { + return 0; + } else { + return -1; + } + } +} + +function shouldIgnoreElement(element) { + var elStyle = getComputedStyle(element); + + if (elStyle) { + var display = elStyle.getPropertyValue("display"); + + if (display === "none") { + return true; + } // Cannot be relied upon, because web browser engine reports invisible when out of view in + // scrolled columns! + // const visibility = elStyle.getPropertyValue("visibility"); + // if (visibility === "hidden") { + // return false; + // } + + + var opacity = elStyle.getPropertyValue("opacity"); + + if (opacity === "0") { + return true; + } + } + + return false; +} + +/***/ }), + /***/ "./src/gestures.js": /*!*************************!*\ !*** ./src/gestures.js ***! @@ -1697,8 +1823,9 @@ function nearestInteractiveElement(element) { "use strict"; __webpack_require__.r(__webpack_exports__); /* harmony import */ var _gestures__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./gestures */ "./src/gestures.js"); -/* harmony import */ var _utils__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./utils */ "./src/utils.js"); -/* harmony import */ var _decorator__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./decorator */ "./src/decorator.js"); +/* harmony import */ var _dom__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./dom */ "./src/dom.js"); +/* harmony import */ var _utils__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./utils */ "./src/utils.js"); +/* harmony import */ var _decorator__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! ./decorator */ "./src/decorator.js"); // // Copyright 2021 Readium Foundation. All rights reserved. // Use of this source code is governed by the BSD-style license @@ -1707,20 +1834,23 @@ __webpack_require__.r(__webpack_exports__); // Base script used by both reflowable and fixed layout resources. + // Public API used by the navigator. __webpack_require__.g.readium = { // utils - scrollToId: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollToId, - scrollToPosition: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollToPosition, - scrollToText: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollToText, - scrollLeft: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollLeft, - scrollRight: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollRight, - setProperty: _utils__WEBPACK_IMPORTED_MODULE_1__.setProperty, - removeProperty: _utils__WEBPACK_IMPORTED_MODULE_1__.removeProperty, + scrollToId: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollToId, + scrollToPosition: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollToPosition, + scrollToText: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollToText, + scrollLeft: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollLeft, + scrollRight: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollRight, + setProperty: _utils__WEBPACK_IMPORTED_MODULE_2__.setProperty, + removeProperty: _utils__WEBPACK_IMPORTED_MODULE_2__.removeProperty, // decoration - registerDecorationTemplates: _decorator__WEBPACK_IMPORTED_MODULE_2__.registerTemplates, - getDecorations: _decorator__WEBPACK_IMPORTED_MODULE_2__.getDecorations + registerDecorationTemplates: _decorator__WEBPACK_IMPORTED_MODULE_3__.registerTemplates, + getDecorations: _decorator__WEBPACK_IMPORTED_MODULE_3__.getDecorations, + // DOM + findFirstVisibleLocator: _dom__WEBPACK_IMPORTED_MODULE_1__.findFirstVisibleLocator }; /***/ }), @@ -2481,12 +2611,11 @@ function scrollToText(text) { return false; } - scrollToRange(range); - return true; + return scrollToRange(range); } function scrollToRange(range) { - scrollToRect(range.getBoundingClientRect()); + return scrollToRect(range.getBoundingClientRect()); } function scrollToRect(rect) { @@ -2495,6 +2624,8 @@ function scrollToRect(rect) { } else { document.scrollingElement.scrollLeft = snapOffset(rect.left + window.scrollX); } + + return true; } // Returns false if the page is already at the left-most scroll offset. @@ -2550,7 +2681,18 @@ function rangeFromLocator(locator) { } try { - var anchor = new _vendor_hypothesis_anchoring_types__WEBPACK_IMPORTED_MODULE_0__.TextQuoteAnchor(document.body, text.highlight, { + var root; + var locations = locator.locations; + + if (locations && locations.cssSelector) { + root = document.querySelector(locations.cssSelector); + } + + if (!root) { + root = document.body; + } + + var anchor = new _vendor_hypothesis_anchoring_types__WEBPACK_IMPORTED_MODULE_0__.TextQuoteAnchor(root, text.highlight, { prefix: text.before, suffix: text.after }); @@ -6029,6 +6171,16 @@ module.exports = function shimMatchAll() { /***/ }), +/***/ "./node_modules/css-selector-generator/build/index.js": +/*!************************************************************!*\ + !*** ./node_modules/css-selector-generator/build/index.js ***! + \************************************************************/ +/***/ ((module) => { + +!function(t,e){ true?module.exports=e():0}(self,(()=>(()=>{var t={426:(t,e,n)=>{var r=n(529);function o(t,e,n){Array.isArray(t)?t.push(e):t[n]=e}t.exports=function(t){var e,n,i,u=[];if(Array.isArray(t))n=[],e=t.length-1;else{if("object"!=typeof t||null===t)throw new TypeError("Expecting an Array or an Object, but `"+(null===t?"null":typeof t)+"` provided.");n={},i=Object.keys(t),e=i.length-1}return function n(c,a){var l,s,f,d;for(s=i?i[a]:a,Array.isArray(t[s])||(void 0===t[s]?t[s]=[]:t[s]=[t[s]]),l=0;l=e?u.push(f):n(f,a+1)}(n,0),u}},529:t=>{t.exports=function(){for(var t={},n=0;n{var e=t&&t.__esModule?()=>t.default:()=>t;return n.d(e,{a:e}),e},n.d=(t,e)=>{for(var r in e)n.o(e,r)&&!n.o(t,r)&&Object.defineProperty(t,r,{enumerable:!0,get:e[r]})},n.o=(t,e)=>Object.prototype.hasOwnProperty.call(t,e),n.r=t=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})};var r={};return(()=>{"use strict";n.r(r),n.d(r,{default:()=>Q,getCssSelector:()=>K});var t,e,o="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t};function i(t){return null!=t&&"object"===(void 0===t?"undefined":o(t))&&1===t.nodeType&&"object"===o(t.style)&&"object"===o(t.ownerDocument)}function u(t="unknown problem",...e){console.warn(`CssSelectorGenerator: ${t}`,...e)}!function(t){t.NONE="none",t.DESCENDANT="descendant",t.CHILD="child"}(t||(t={})),function(t){t.id="id",t.class="class",t.tag="tag",t.attribute="attribute",t.nthchild="nthchild",t.nthoftype="nthoftype"}(e||(e={}));const c={selectors:[e.id,e.class,e.tag,e.attribute],includeTag:!1,whitelist:[],blacklist:[],combineWithinSelector:!0,combineBetweenSelectors:!0,root:null,maxCombinations:Number.POSITIVE_INFINITY,maxCandidates:Number.POSITIVE_INFINITY};function a(t){return t instanceof RegExp}function l(t){return["string","function"].includes(typeof t)||a(t)}function s(t){return Array.isArray(t)?t.filter(l):[]}function f(t){const e=[Node.DOCUMENT_NODE,Node.DOCUMENT_FRAGMENT_NODE,Node.ELEMENT_NODE];return function(t){return t instanceof Node}(t)&&e.includes(t.nodeType)}function d(t,e){if(f(t))return t.contains(e)||u("element root mismatch","Provided root does not contain the element. This will most likely result in producing a fallback selector using element's real root node. If you plan to use the selector using provided root (e.g. `root.querySelector`), it will nto work as intended."),t;const n=e.getRootNode({composed:!1});return f(n)?(n!==document&&u("shadow root inferred","You did not provide a root and the element is a child of Shadow DOM. This will produce a selector using ShadowRoot as a root. If you plan to use the selector using document as a root (e.g. `document.querySelector`), it will not work as intended."),n):e.ownerDocument.querySelector(":root")}function p(t){return"number"==typeof t?t:Number.POSITIVE_INFINITY}function m(t=[]){const[e=[],...n]=t;return 0===n.length?e:n.reduce(((t,e)=>t.filter((t=>e.includes(t)))),e)}function h(t){return[].concat(...t)}function y(t){const e=t.map((t=>{if(a(t))return e=>t.test(e);if("function"==typeof t)return e=>{const n=t(e);return"boolean"!=typeof n?(u("pattern matcher function invalid","Provided pattern matching function does not return boolean. It's result will be ignored.",t),!1):n};if("string"==typeof t){const e=new RegExp("^"+t.replace(/[|\\{}()[\]^$+?.]/g,"\\$&").replace(/\*/g,".+")+"$");return t=>e.test(t)}return u("pattern matcher invalid","Pattern matching only accepts strings, regular expressions and/or functions. This item is invalid and will be ignored.",t),()=>!1}));return t=>e.some((e=>e(t)))}function g(t,e,n){const r=Array.from(d(n,t[0]).querySelectorAll(e));return r.length===t.length&&t.every((t=>r.includes(t)))}function b(t,e){e=null!=e?e:function(t){return t.ownerDocument.querySelector(":root")}(t);const n=[];let r=t;for(;i(r)&&r!==e;)n.push(r),r=r.parentElement;return n}function v(t,e){return m(t.map((t=>b(t,e))))}const N={[t.NONE]:{type:t.NONE,value:""},[t.DESCENDANT]:{type:t.DESCENDANT,value:" > "},[t.CHILD]:{type:t.CHILD,value:" "}},S=new RegExp(["^$","\\s","^\\d"].join("|")),E=new RegExp(["^$","^\\d"].join("|")),w=[e.nthoftype,e.tag,e.id,e.class,e.attribute,e.nthchild];var x=n(426),A=n.n(x);const C=y(["class","id","ng-*"]);function O({nodeName:t}){return`[${t}]`}function T({nodeName:t,nodeValue:e}){return`[${t}='${V(e)}']`}function I(t){const e=Array.from(t.attributes).filter((e=>function({nodeName:t},e){const n=e.tagName.toLowerCase();return!(["input","option"].includes(n)&&"value"===t||C(t))}(e,t)));return[...e.map(O),...e.map(T)]}function j(t){return(t.getAttribute("class")||"").trim().split(/\s+/).filter((t=>!E.test(t))).map((t=>`.${V(t)}`))}function D(t){const e=t.getAttribute("id")||"",n=`#${V(e)}`,r=t.getRootNode({composed:!1});return!S.test(e)&&g([t],n,r)?[n]:[]}function $(t){const e=t.parentNode;if(e){const n=Array.from(e.childNodes).filter(i).indexOf(t);if(n>-1)return[`:nth-child(${n+1})`]}return[]}function P(t){return[V(t.tagName.toLowerCase())]}function R(t){const e=[...new Set(h(t.map(P)))];return 0===e.length||e.length>1?[]:[e[0]]}function _(t){const e=R([t])[0],n=t.parentElement;if(n){const r=Array.from(n.children).filter((t=>t.tagName.toLowerCase()===e)).indexOf(t);if(r>-1)return[`${e}:nth-of-type(${r+1})`]}return[]}function k(t=[],{maxResults:e=Number.POSITIVE_INFINITY}={}){const n=[];let r=0,o=L(1);for(;o.length<=t.length&&rt[e]))),o=M(o,t.length-1);return n}function M(t=[],e=0){const n=t.length;if(0===n)return[];const r=[...t];r[n-1]+=1;for(let t=n-1;t>=0;t--)if(r[t]>e){if(0===t)return L(n+1);r[t-1]++,r[t]=r[t-1]+1}return r[n-1]>e?L(n+1):r}function L(t=1){return Array.from(Array(t).keys())}const q=":".charCodeAt(0).toString(16).toUpperCase(),F=/[ !"#$%&'()\[\]{|}<>*+,./;=?@^`~\\]/;function V(t=""){var e,n;return null!==(n=null===(e=null===CSS||void 0===CSS?void 0:CSS.escape)||void 0===e?void 0:e.call(CSS,t))&&void 0!==n?n:function(t=""){return t.split("").map((t=>":"===t?`\\${q} `:F.test(t)?`\\${t}`:escape(t).replace(/%/g,"\\"))).join("")}(t)}const Y={tag:R,id:function(t){return 0===t.length||t.length>1?[]:D(t[0])},class:function(t){return m(t.map(j))},attribute:function(t){return m(t.map(I))},nthchild:function(t){return m(t.map($))},nthoftype:function(t){return m(t.map(_))}},B={tag:P,id:D,class:j,attribute:I,nthchild:$,nthoftype:_};function G(t){return t.includes(e.tag)||t.includes(e.nthoftype)?[...t]:[...t,e.tag]}function W(t={}){const n=[...w];return t[e.tag]&&t[e.nthoftype]&&n.splice(n.indexOf(e.tag),1),n.map((e=>{return(r=t)[n=e]?r[n].join(""):"";var n,r})).join("")}function H(t,e,n="",r){const o=function(t,e){return""===e?t:function(t,e){return[...t.map((t=>e+" "+t)),...t.map((t=>e+" > "+t))]}(t,e)}(function(t,e,n){const r=h(function(t,e){return function(t){const{selectors:e,combineBetweenSelectors:n,includeTag:r,maxCandidates:o}=t,i=n?k(e,{maxResults:o}):e.map((t=>[t]));return r?i.map(G):i}(e).map((e=>function(t,e){const n={};return t.forEach((t=>{const r=e[t];r.length>0&&(n[t]=r)})),A()(n).map(W)}(e,t))).filter((t=>t.length>0))}(function(t,e){const{blacklist:n,whitelist:r,combineWithinSelector:o,maxCombinations:i}=e,u=y(n),c=y(r);return function(t){const{selectors:e,includeTag:n}=t,r=[].concat(e);return n&&!r.includes("tag")&&r.push("tag"),r}(e).reduce(((e,n)=>{const r=function(t=[],e){return t.sort(((t,n)=>{const r=e(t),o=e(n);return r&&!o?-1:!r&&o?1:0}))}(function(t=[],e,n){return t.filter((t=>n(t)||!e(t)))}(function(t,e){var n;return(null!==(n=Y[e])&&void 0!==n?n:()=>[])(t)}(t,n),u,c),c);return e[n]=o?k(r,{maxResults:i}):r.map((t=>[t])),e}),{})}(t,n),n));return[...new Set(r)]}(t,r.root,r),n);for(const e of o)if(g(t,e,r.root))return e;return null}function U(t){return{value:t,include:!1}}function z({selectors:t,operator:n}){let r=[...w];t[e.tag]&&t[e.nthoftype]&&(r=r.filter((t=>t!==e.tag)));let o="";return r.forEach((e=>{(t[e]||[]).forEach((({value:t,include:e})=>{e&&(o+=t)}))})),n.value+o}function J(n){return[":root",...b(n).reverse().map((n=>{const r=function(e,n,r=t.NONE){const o={};return n.forEach((t=>{Reflect.set(o,t,function(t,e){return B[e](t)}(e,t).map(U))})),{element:e,operator:N[r],selectors:o}}(n,[e.nthchild],t.DESCENDANT);return r.selectors.nthchild.forEach((t=>{t.include=!0})),r})).map(z)].join("")}function K(t,n={}){const r=function(t){const e=(Array.isArray(t)?t:[t]).filter(i);return[...new Set(e)]}(t),o=function(t,n={}){const r=Object.assign(Object.assign({},c),n);return{selectors:(o=r.selectors,Array.isArray(o)?o.filter((t=>{return n=e,r=t,Object.values(n).includes(r);var n,r})):[]),whitelist:s(r.whitelist),blacklist:s(r.blacklist),root:d(r.root,t),combineWithinSelector:!!r.combineWithinSelector,combineBetweenSelectors:!!r.combineBetweenSelectors,includeTag:!!r.includeTag,maxCombinations:p(r.maxCombinations),maxCandidates:p(r.maxCandidates)};var o}(r[0],n);let u="",a=o.root;function l(){return function(t,e,n="",r){if(0===t.length)return null;const o=[t.length>1?t:[],...v(t,e).map((t=>[t]))];for(const t of o){const e=H(t,0,n,r);if(e)return{foundElements:t,selector:e}}return null}(r,a,u,o)}let f=l();for(;f;){const{foundElements:t,selector:e}=f;if(g(r,e,o.root))return e;a=t[0],u=e,f=l()}return r.length>1?r.map((t=>K(t,o))).join(", "):function(t){return t.map(J).join(", ")}(r)}const Q=K})(),r})())); + +/***/ }), + /***/ "./node_modules/es-abstract/2020/IsArray.js": /*!**************************************************!*\ !*** ./node_modules/es-abstract/2020/IsArray.js ***! @@ -8104,6 +8256,7 @@ __webpack_require__.r(__webpack_exports__); // // Script used for fixed layouts resources. +window.readium.isFixedLayout = true; })(); /******/ })() diff --git a/Sources/Navigator/EPUB/Assets/Static/scripts/readium-reflowable.js b/Sources/Navigator/EPUB/Assets/Static/scripts/readium-reflowable.js index e3da98ce7..2b58126cb 100644 --- a/Sources/Navigator/EPUB/Assets/Static/scripts/readium-reflowable.js +++ b/Sources/Navigator/EPUB/Assets/Static/scripts/readium-reflowable.js @@ -1612,6 +1612,132 @@ window.addEventListener("load", function () { /***/ }), +/***/ "./src/dom.js": +/*!********************!*\ + !*** ./src/dom.js ***! + \********************/ +/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { + +"use strict"; +__webpack_require__.r(__webpack_exports__); +/* harmony export */ __webpack_require__.d(__webpack_exports__, { +/* harmony export */ "findFirstVisibleLocator": () => (/* binding */ findFirstVisibleLocator) +/* harmony export */ }); +/* harmony import */ var _utils__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./utils */ "./src/utils.js"); +/* harmony import */ var css_selector_generator__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! css-selector-generator */ "./node_modules/css-selector-generator/build/index.js"); +/* harmony import */ var css_selector_generator__WEBPACK_IMPORTED_MODULE_1___default = /*#__PURE__*/__webpack_require__.n(css_selector_generator__WEBPACK_IMPORTED_MODULE_1__); +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + + +function findFirstVisibleLocator() { + var element = findElement(document.body); + + if (!element) { + return undefined; + } + + return { + href: "#", + type: "application/xhtml+xml", + locations: { + cssSelector: (0,css_selector_generator__WEBPACK_IMPORTED_MODULE_1__.getCssSelector)(element) + }, + text: { + highlight: element.textContent + } + }; +} + +function findElement(rootElement) { + var foundElement = undefined; + + for (var i = rootElement.children.length - 1; i >= 0; i--) { + var child = rootElement.children[i]; + var position = elementRelativePosition(child, undefined); + + if (position == 0) { + if (!shouldIgnoreElement(child)) { + foundElement = child; + } + } else if (position < 0) { + if (!foundElement) { + foundElement = child; + } + + break; + } + } + + if (foundElement) { + return findElement(foundElement); + } + + return rootElement; +} // See computeVisibility_() in r2-navigator-js + + +function elementRelativePosition(element, domRect +/* nullable */ +) { + if (readium.isFixedLayout) return true; + + if (element === document.body || element === document.documentElement) { + return -1; + } + + if (!document || !document.documentElement || !document.body) { + return 1; + } + + var rect = domRect || element.getBoundingClientRect(); + + if ((0,_utils__WEBPACK_IMPORTED_MODULE_0__.isScrollModeEnabled)()) { + return rect.top >= 0 && rect.top <= document.documentElement.clientHeight; + } else { + var pageWidth = window.innerWidth; + + if (rect.left >= pageWidth) { + return 1; + } else if (rect.left >= 0) { + return 0; + } else { + return -1; + } + } +} + +function shouldIgnoreElement(element) { + var elStyle = getComputedStyle(element); + + if (elStyle) { + var display = elStyle.getPropertyValue("display"); + + if (display === "none") { + return true; + } // Cannot be relied upon, because web browser engine reports invisible when out of view in + // scrolled columns! + // const visibility = elStyle.getPropertyValue("visibility"); + // if (visibility === "hidden") { + // return false; + // } + + + var opacity = elStyle.getPropertyValue("opacity"); + + if (opacity === "0") { + return true; + } + } + + return false; +} + +/***/ }), + /***/ "./src/gestures.js": /*!*************************!*\ !*** ./src/gestures.js ***! @@ -1697,8 +1823,9 @@ function nearestInteractiveElement(element) { "use strict"; __webpack_require__.r(__webpack_exports__); /* harmony import */ var _gestures__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./gestures */ "./src/gestures.js"); -/* harmony import */ var _utils__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./utils */ "./src/utils.js"); -/* harmony import */ var _decorator__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./decorator */ "./src/decorator.js"); +/* harmony import */ var _dom__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./dom */ "./src/dom.js"); +/* harmony import */ var _utils__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./utils */ "./src/utils.js"); +/* harmony import */ var _decorator__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! ./decorator */ "./src/decorator.js"); // // Copyright 2021 Readium Foundation. All rights reserved. // Use of this source code is governed by the BSD-style license @@ -1707,20 +1834,23 @@ __webpack_require__.r(__webpack_exports__); // Base script used by both reflowable and fixed layout resources. + // Public API used by the navigator. __webpack_require__.g.readium = { // utils - scrollToId: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollToId, - scrollToPosition: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollToPosition, - scrollToText: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollToText, - scrollLeft: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollLeft, - scrollRight: _utils__WEBPACK_IMPORTED_MODULE_1__.scrollRight, - setProperty: _utils__WEBPACK_IMPORTED_MODULE_1__.setProperty, - removeProperty: _utils__WEBPACK_IMPORTED_MODULE_1__.removeProperty, + scrollToId: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollToId, + scrollToPosition: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollToPosition, + scrollToText: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollToText, + scrollLeft: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollLeft, + scrollRight: _utils__WEBPACK_IMPORTED_MODULE_2__.scrollRight, + setProperty: _utils__WEBPACK_IMPORTED_MODULE_2__.setProperty, + removeProperty: _utils__WEBPACK_IMPORTED_MODULE_2__.removeProperty, // decoration - registerDecorationTemplates: _decorator__WEBPACK_IMPORTED_MODULE_2__.registerTemplates, - getDecorations: _decorator__WEBPACK_IMPORTED_MODULE_2__.getDecorations + registerDecorationTemplates: _decorator__WEBPACK_IMPORTED_MODULE_3__.registerTemplates, + getDecorations: _decorator__WEBPACK_IMPORTED_MODULE_3__.getDecorations, + // DOM + findFirstVisibleLocator: _dom__WEBPACK_IMPORTED_MODULE_1__.findFirstVisibleLocator }; /***/ }), @@ -2481,12 +2611,11 @@ function scrollToText(text) { return false; } - scrollToRange(range); - return true; + return scrollToRange(range); } function scrollToRange(range) { - scrollToRect(range.getBoundingClientRect()); + return scrollToRect(range.getBoundingClientRect()); } function scrollToRect(rect) { @@ -2495,6 +2624,8 @@ function scrollToRect(rect) { } else { document.scrollingElement.scrollLeft = snapOffset(rect.left + window.scrollX); } + + return true; } // Returns false if the page is already at the left-most scroll offset. @@ -2550,7 +2681,18 @@ function rangeFromLocator(locator) { } try { - var anchor = new _vendor_hypothesis_anchoring_types__WEBPACK_IMPORTED_MODULE_0__.TextQuoteAnchor(document.body, text.highlight, { + var root; + var locations = locator.locations; + + if (locations && locations.cssSelector) { + root = document.querySelector(locations.cssSelector); + } + + if (!root) { + root = document.body; + } + + var anchor = new _vendor_hypothesis_anchoring_types__WEBPACK_IMPORTED_MODULE_0__.TextQuoteAnchor(root, text.highlight, { prefix: text.before, suffix: text.after }); @@ -6029,6 +6171,16 @@ module.exports = function shimMatchAll() { /***/ }), +/***/ "./node_modules/css-selector-generator/build/index.js": +/*!************************************************************!*\ + !*** ./node_modules/css-selector-generator/build/index.js ***! + \************************************************************/ +/***/ ((module) => { + +!function(t,e){ true?module.exports=e():0}(self,(()=>(()=>{var t={426:(t,e,n)=>{var r=n(529);function o(t,e,n){Array.isArray(t)?t.push(e):t[n]=e}t.exports=function(t){var e,n,i,u=[];if(Array.isArray(t))n=[],e=t.length-1;else{if("object"!=typeof t||null===t)throw new TypeError("Expecting an Array or an Object, but `"+(null===t?"null":typeof t)+"` provided.");n={},i=Object.keys(t),e=i.length-1}return function n(c,a){var l,s,f,d;for(s=i?i[a]:a,Array.isArray(t[s])||(void 0===t[s]?t[s]=[]:t[s]=[t[s]]),l=0;l=e?u.push(f):n(f,a+1)}(n,0),u}},529:t=>{t.exports=function(){for(var t={},n=0;n{var e=t&&t.__esModule?()=>t.default:()=>t;return n.d(e,{a:e}),e},n.d=(t,e)=>{for(var r in e)n.o(e,r)&&!n.o(t,r)&&Object.defineProperty(t,r,{enumerable:!0,get:e[r]})},n.o=(t,e)=>Object.prototype.hasOwnProperty.call(t,e),n.r=t=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})};var r={};return(()=>{"use strict";n.r(r),n.d(r,{default:()=>Q,getCssSelector:()=>K});var t,e,o="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t};function i(t){return null!=t&&"object"===(void 0===t?"undefined":o(t))&&1===t.nodeType&&"object"===o(t.style)&&"object"===o(t.ownerDocument)}function u(t="unknown problem",...e){console.warn(`CssSelectorGenerator: ${t}`,...e)}!function(t){t.NONE="none",t.DESCENDANT="descendant",t.CHILD="child"}(t||(t={})),function(t){t.id="id",t.class="class",t.tag="tag",t.attribute="attribute",t.nthchild="nthchild",t.nthoftype="nthoftype"}(e||(e={}));const c={selectors:[e.id,e.class,e.tag,e.attribute],includeTag:!1,whitelist:[],blacklist:[],combineWithinSelector:!0,combineBetweenSelectors:!0,root:null,maxCombinations:Number.POSITIVE_INFINITY,maxCandidates:Number.POSITIVE_INFINITY};function a(t){return t instanceof RegExp}function l(t){return["string","function"].includes(typeof t)||a(t)}function s(t){return Array.isArray(t)?t.filter(l):[]}function f(t){const e=[Node.DOCUMENT_NODE,Node.DOCUMENT_FRAGMENT_NODE,Node.ELEMENT_NODE];return function(t){return t instanceof Node}(t)&&e.includes(t.nodeType)}function d(t,e){if(f(t))return t.contains(e)||u("element root mismatch","Provided root does not contain the element. This will most likely result in producing a fallback selector using element's real root node. If you plan to use the selector using provided root (e.g. `root.querySelector`), it will nto work as intended."),t;const n=e.getRootNode({composed:!1});return f(n)?(n!==document&&u("shadow root inferred","You did not provide a root and the element is a child of Shadow DOM. This will produce a selector using ShadowRoot as a root. If you plan to use the selector using document as a root (e.g. `document.querySelector`), it will not work as intended."),n):e.ownerDocument.querySelector(":root")}function p(t){return"number"==typeof t?t:Number.POSITIVE_INFINITY}function m(t=[]){const[e=[],...n]=t;return 0===n.length?e:n.reduce(((t,e)=>t.filter((t=>e.includes(t)))),e)}function h(t){return[].concat(...t)}function y(t){const e=t.map((t=>{if(a(t))return e=>t.test(e);if("function"==typeof t)return e=>{const n=t(e);return"boolean"!=typeof n?(u("pattern matcher function invalid","Provided pattern matching function does not return boolean. It's result will be ignored.",t),!1):n};if("string"==typeof t){const e=new RegExp("^"+t.replace(/[|\\{}()[\]^$+?.]/g,"\\$&").replace(/\*/g,".+")+"$");return t=>e.test(t)}return u("pattern matcher invalid","Pattern matching only accepts strings, regular expressions and/or functions. This item is invalid and will be ignored.",t),()=>!1}));return t=>e.some((e=>e(t)))}function g(t,e,n){const r=Array.from(d(n,t[0]).querySelectorAll(e));return r.length===t.length&&t.every((t=>r.includes(t)))}function b(t,e){e=null!=e?e:function(t){return t.ownerDocument.querySelector(":root")}(t);const n=[];let r=t;for(;i(r)&&r!==e;)n.push(r),r=r.parentElement;return n}function v(t,e){return m(t.map((t=>b(t,e))))}const N={[t.NONE]:{type:t.NONE,value:""},[t.DESCENDANT]:{type:t.DESCENDANT,value:" > "},[t.CHILD]:{type:t.CHILD,value:" "}},S=new RegExp(["^$","\\s","^\\d"].join("|")),E=new RegExp(["^$","^\\d"].join("|")),w=[e.nthoftype,e.tag,e.id,e.class,e.attribute,e.nthchild];var x=n(426),A=n.n(x);const C=y(["class","id","ng-*"]);function O({nodeName:t}){return`[${t}]`}function T({nodeName:t,nodeValue:e}){return`[${t}='${V(e)}']`}function I(t){const e=Array.from(t.attributes).filter((e=>function({nodeName:t},e){const n=e.tagName.toLowerCase();return!(["input","option"].includes(n)&&"value"===t||C(t))}(e,t)));return[...e.map(O),...e.map(T)]}function j(t){return(t.getAttribute("class")||"").trim().split(/\s+/).filter((t=>!E.test(t))).map((t=>`.${V(t)}`))}function D(t){const e=t.getAttribute("id")||"",n=`#${V(e)}`,r=t.getRootNode({composed:!1});return!S.test(e)&&g([t],n,r)?[n]:[]}function $(t){const e=t.parentNode;if(e){const n=Array.from(e.childNodes).filter(i).indexOf(t);if(n>-1)return[`:nth-child(${n+1})`]}return[]}function P(t){return[V(t.tagName.toLowerCase())]}function R(t){const e=[...new Set(h(t.map(P)))];return 0===e.length||e.length>1?[]:[e[0]]}function _(t){const e=R([t])[0],n=t.parentElement;if(n){const r=Array.from(n.children).filter((t=>t.tagName.toLowerCase()===e)).indexOf(t);if(r>-1)return[`${e}:nth-of-type(${r+1})`]}return[]}function k(t=[],{maxResults:e=Number.POSITIVE_INFINITY}={}){const n=[];let r=0,o=L(1);for(;o.length<=t.length&&rt[e]))),o=M(o,t.length-1);return n}function M(t=[],e=0){const n=t.length;if(0===n)return[];const r=[...t];r[n-1]+=1;for(let t=n-1;t>=0;t--)if(r[t]>e){if(0===t)return L(n+1);r[t-1]++,r[t]=r[t-1]+1}return r[n-1]>e?L(n+1):r}function L(t=1){return Array.from(Array(t).keys())}const q=":".charCodeAt(0).toString(16).toUpperCase(),F=/[ !"#$%&'()\[\]{|}<>*+,./;=?@^`~\\]/;function V(t=""){var e,n;return null!==(n=null===(e=null===CSS||void 0===CSS?void 0:CSS.escape)||void 0===e?void 0:e.call(CSS,t))&&void 0!==n?n:function(t=""){return t.split("").map((t=>":"===t?`\\${q} `:F.test(t)?`\\${t}`:escape(t).replace(/%/g,"\\"))).join("")}(t)}const Y={tag:R,id:function(t){return 0===t.length||t.length>1?[]:D(t[0])},class:function(t){return m(t.map(j))},attribute:function(t){return m(t.map(I))},nthchild:function(t){return m(t.map($))},nthoftype:function(t){return m(t.map(_))}},B={tag:P,id:D,class:j,attribute:I,nthchild:$,nthoftype:_};function G(t){return t.includes(e.tag)||t.includes(e.nthoftype)?[...t]:[...t,e.tag]}function W(t={}){const n=[...w];return t[e.tag]&&t[e.nthoftype]&&n.splice(n.indexOf(e.tag),1),n.map((e=>{return(r=t)[n=e]?r[n].join(""):"";var n,r})).join("")}function H(t,e,n="",r){const o=function(t,e){return""===e?t:function(t,e){return[...t.map((t=>e+" "+t)),...t.map((t=>e+" > "+t))]}(t,e)}(function(t,e,n){const r=h(function(t,e){return function(t){const{selectors:e,combineBetweenSelectors:n,includeTag:r,maxCandidates:o}=t,i=n?k(e,{maxResults:o}):e.map((t=>[t]));return r?i.map(G):i}(e).map((e=>function(t,e){const n={};return t.forEach((t=>{const r=e[t];r.length>0&&(n[t]=r)})),A()(n).map(W)}(e,t))).filter((t=>t.length>0))}(function(t,e){const{blacklist:n,whitelist:r,combineWithinSelector:o,maxCombinations:i}=e,u=y(n),c=y(r);return function(t){const{selectors:e,includeTag:n}=t,r=[].concat(e);return n&&!r.includes("tag")&&r.push("tag"),r}(e).reduce(((e,n)=>{const r=function(t=[],e){return t.sort(((t,n)=>{const r=e(t),o=e(n);return r&&!o?-1:!r&&o?1:0}))}(function(t=[],e,n){return t.filter((t=>n(t)||!e(t)))}(function(t,e){var n;return(null!==(n=Y[e])&&void 0!==n?n:()=>[])(t)}(t,n),u,c),c);return e[n]=o?k(r,{maxResults:i}):r.map((t=>[t])),e}),{})}(t,n),n));return[...new Set(r)]}(t,r.root,r),n);for(const e of o)if(g(t,e,r.root))return e;return null}function U(t){return{value:t,include:!1}}function z({selectors:t,operator:n}){let r=[...w];t[e.tag]&&t[e.nthoftype]&&(r=r.filter((t=>t!==e.tag)));let o="";return r.forEach((e=>{(t[e]||[]).forEach((({value:t,include:e})=>{e&&(o+=t)}))})),n.value+o}function J(n){return[":root",...b(n).reverse().map((n=>{const r=function(e,n,r=t.NONE){const o={};return n.forEach((t=>{Reflect.set(o,t,function(t,e){return B[e](t)}(e,t).map(U))})),{element:e,operator:N[r],selectors:o}}(n,[e.nthchild],t.DESCENDANT);return r.selectors.nthchild.forEach((t=>{t.include=!0})),r})).map(z)].join("")}function K(t,n={}){const r=function(t){const e=(Array.isArray(t)?t:[t]).filter(i);return[...new Set(e)]}(t),o=function(t,n={}){const r=Object.assign(Object.assign({},c),n);return{selectors:(o=r.selectors,Array.isArray(o)?o.filter((t=>{return n=e,r=t,Object.values(n).includes(r);var n,r})):[]),whitelist:s(r.whitelist),blacklist:s(r.blacklist),root:d(r.root,t),combineWithinSelector:!!r.combineWithinSelector,combineBetweenSelectors:!!r.combineBetweenSelectors,includeTag:!!r.includeTag,maxCombinations:p(r.maxCombinations),maxCandidates:p(r.maxCandidates)};var o}(r[0],n);let u="",a=o.root;function l(){return function(t,e,n="",r){if(0===t.length)return null;const o=[t.length>1?t:[],...v(t,e).map((t=>[t]))];for(const t of o){const e=H(t,0,n,r);if(e)return{foundElements:t,selector:e}}return null}(r,a,u,o)}let f=l();for(;f;){const{foundElements:t,selector:e}=f;if(g(r,e,o.root))return e;a=t[0],u=e,f=l()}return r.length>1?r.map((t=>K(t,o))).join(", "):function(t){return t.map(J).join(", ")}(r)}const Q=K})(),r})())); + +/***/ }), + /***/ "./node_modules/es-abstract/2020/IsArray.js": /*!**************************************************!*\ !*** ./node_modules/es-abstract/2020/IsArray.js ***! @@ -8104,6 +8256,7 @@ __webpack_require__.r(__webpack_exports__); // // Script used for reflowable resources. +window.readium.isReflowable = true; window.addEventListener("load", function () { // Notifies native code that the page is loaded after it is rendered. // Waiting for the next animation frame seems to do the trick to make sure the page is fully rendered. diff --git a/Sources/Navigator/EPUB/Scripts/package.json b/Sources/Navigator/EPUB/Scripts/package.json index 912d80f29..e3fc29c51 100644 --- a/Sources/Navigator/EPUB/Scripts/package.json +++ b/Sources/Navigator/EPUB/Scripts/package.json @@ -11,7 +11,9 @@ "checkformat": "prettier --check '**/*.js'", "format": "prettier --list-different --write '**/*.js'" }, - "browserslist": [ "iOS 10" ], + "browserslist": [ + "iOS 10" + ], "devDependencies": { "@babel/core": "^7.16.0", "@babel/preset-env": "^7.16.0", @@ -24,6 +26,7 @@ "dependencies": { "@juggle/resize-observer": "^3.3.1", "approx-string-match": "^1.1.0", + "css-selector-generator": "^3.6.1", "string.prototype.matchall": "^4.0.5" } } diff --git a/Sources/Navigator/EPUB/Scripts/src/dom.js b/Sources/Navigator/EPUB/Scripts/src/dom.js new file mode 100644 index 000000000..84ed769e8 --- /dev/null +++ b/Sources/Navigator/EPUB/Scripts/src/dom.js @@ -0,0 +1,98 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import { isScrollModeEnabled } from "./utils"; +import { getCssSelector } from "css-selector-generator"; + +export function findFirstVisibleLocator() { + const element = findElement(document.body); + if (!element) { + return undefined; + } + + return { + href: "#", + type: "application/xhtml+xml", + locations: { + cssSelector: getCssSelector(element), + }, + text: { + highlight: element.textContent, + }, + }; +} + +function findElement(rootElement) { + var foundElement = undefined; + for (var i = rootElement.children.length - 1; i >= 0; i--) { + const child = rootElement.children[i]; + const position = elementRelativePosition(child, undefined); + if (position == 0) { + if (!shouldIgnoreElement(child)) { + foundElement = child; + } + } else if (position < 0) { + if (!foundElement) { + foundElement = child; + } + break; + } + } + + if (foundElement) { + return findElement(foundElement); + } + return rootElement; +} + +// See computeVisibility_() in r2-navigator-js +function elementRelativePosition(element, domRect /* nullable */) { + if (readium.isFixedLayout) return true; + + if (element === document.body || element === document.documentElement) { + return -1; + } + if (!document || !document.documentElement || !document.body) { + return 1; + } + + const rect = domRect || element.getBoundingClientRect(); + + if (isScrollModeEnabled()) { + return rect.top >= 0 && rect.top <= document.documentElement.clientHeight; + } else { + const pageWidth = window.innerWidth; + if (rect.left >= pageWidth) { + return 1; + } else if (rect.left >= 0) { + return 0; + } else { + return -1; + } + } +} + +function shouldIgnoreElement(element) { + const elStyle = getComputedStyle(element); + if (elStyle) { + const display = elStyle.getPropertyValue("display"); + if (display === "none") { + return true; + } + // Cannot be relied upon, because web browser engine reports invisible when out of view in + // scrolled columns! + // const visibility = elStyle.getPropertyValue("visibility"); + // if (visibility === "hidden") { + // return false; + // } + const opacity = elStyle.getPropertyValue("opacity"); + if (opacity === "0") { + return true; + } + } + + return false; +} diff --git a/Sources/Navigator/EPUB/Scripts/src/index-fixed.js b/Sources/Navigator/EPUB/Scripts/src/index-fixed.js index c868cee96..2f7b0d13b 100644 --- a/Sources/Navigator/EPUB/Scripts/src/index-fixed.js +++ b/Sources/Navigator/EPUB/Scripts/src/index-fixed.js @@ -7,3 +7,5 @@ // Script used for fixed layouts resources. import "./index"; + +window.readium.isFixedLayout = true; diff --git a/Sources/Navigator/EPUB/Scripts/src/index-reflowable.js b/Sources/Navigator/EPUB/Scripts/src/index-reflowable.js index 52b6bc66d..13d857aa8 100644 --- a/Sources/Navigator/EPUB/Scripts/src/index-reflowable.js +++ b/Sources/Navigator/EPUB/Scripts/src/index-reflowable.js @@ -8,6 +8,8 @@ import "./index"; +window.readium.isReflowable = true; + window.addEventListener("load", function () { // Notifies native code that the page is loaded after it is rendered. // Waiting for the next animation frame seems to do the trick to make sure the page is fully rendered. diff --git a/Sources/Navigator/EPUB/Scripts/src/index.js b/Sources/Navigator/EPUB/Scripts/src/index.js index 1743731bd..aa8f65732 100644 --- a/Sources/Navigator/EPUB/Scripts/src/index.js +++ b/Sources/Navigator/EPUB/Scripts/src/index.js @@ -7,6 +7,7 @@ // Base script used by both reflowable and fixed layout resources. import "./gestures"; +import { findFirstVisibleLocator } from "./dom"; import { removeProperty, scrollLeft, @@ -32,4 +33,7 @@ global.readium = { // decoration registerDecorationTemplates: registerTemplates, getDecorations: getDecorations, + + // DOM + findFirstVisibleLocator: findFirstVisibleLocator, }; diff --git a/Sources/Navigator/EPUB/Scripts/src/utils.js b/Sources/Navigator/EPUB/Scripts/src/utils.js index 94e1a3e75..2ec841d40 100644 --- a/Sources/Navigator/EPUB/Scripts/src/utils.js +++ b/Sources/Navigator/EPUB/Scripts/src/utils.js @@ -179,12 +179,11 @@ export function scrollToText(text) { if (!range) { return false; } - scrollToRange(range); - return true; + return scrollToRange(range); } function scrollToRange(range) { - scrollToRect(range.getBoundingClientRect()); + return scrollToRect(range.getBoundingClientRect()); } function scrollToRect(rect) { @@ -196,6 +195,8 @@ function scrollToRect(rect) { rect.left + window.scrollX ); } + + return true; } // Returns false if the page is already at the left-most scroll offset. @@ -252,7 +253,15 @@ export function rangeFromLocator(locator) { return null; } try { - let anchor = new TextQuoteAnchor(document.body, text.highlight, { + var root; + let locations = locator.locations; + if (locations && locations.cssSelector) { + root = document.querySelector(locations.cssSelector); + } + if (!root) { + root = document.body; + } + let anchor = new TextQuoteAnchor(root, text.highlight, { prefix: text.before, suffix: text.after, }); diff --git a/Sources/Navigator/EPUB/Scripts/yarn.lock b/Sources/Navigator/EPUB/Scripts/yarn.lock index 834f008d6..412d06309 100644 --- a/Sources/Navigator/EPUB/Scripts/yarn.lock +++ b/Sources/Navigator/EPUB/Scripts/yarn.lock @@ -1305,6 +1305,13 @@ caniuse-lite@^1.0.30001274: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001280.tgz#066a506046ba4be34cde5f74a08db7a396718fb7" integrity sha512-kFXwYvHe5rix25uwueBxC569o53J6TpnGu0BEEn+6Lhl2vsnAumRFWEBhDft1fwyo6m1r4i+RqA4+163FpeFcA== +cartesian@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/cartesian/-/cartesian-1.0.1.tgz#ae3fc8a63e2ba7e2c4989ce696207457bcae65af" + integrity sha1-rj/Ipj4rp+LEmJzmliB0V7yuZa8= + dependencies: + xtend "^4.0.1" + chalk@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -1409,6 +1416,14 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +css-selector-generator@^3.6.1: + version "3.6.1" + resolved "https://registry.yarnpkg.com/css-selector-generator/-/css-selector-generator-3.6.1.tgz#bfe7ea59cb987cc7597e6f80fd57581e85fa0f65" + integrity sha512-00CD9cAH1bYKi/7UztQmsnp1MJjeBMV7hMAR39wG/jDI2ef2BS9ZqW3gKpBN+vJOxB4rNrQG4xJfmgx5ZFCFHQ== + dependencies: + cartesian "^1.0.1" + iselement "^1.1.4" + debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: version "4.3.2" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" @@ -2005,6 +2020,11 @@ is-weakref@^1.0.1: dependencies: call-bind "^1.0.0" +iselement@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/iselement/-/iselement-1.1.4.tgz#7e55b52a8ebca50a7e2e80e5b8d2840f32353146" + integrity sha1-flW1Ko68pQp+LoDluNKEDzI1MUY= + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -2870,6 +2890,11 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= +xtend@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" + integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== + yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" From 9b047ecb3412be67bddc3e0a747653cdc6d67c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 2 May 2022 16:24:21 +0200 Subject: [PATCH 09/46] Add the `TTSController` --- .../EPUB/EPUBNavigatorViewController.swift | 9 + Sources/Navigator/EPUB/EPUBSpreadView.swift | 18 +- Sources/Navigator/TTS/TTSController.swift | 287 ++++++++++++++++++ .../Services/Content/ContentIterator.swift | 6 +- 4 files changed, 316 insertions(+), 4 deletions(-) create mode 100644 Sources/Navigator/TTS/TTSController.swift diff --git a/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift b/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift index 75eaea02f..47da33c95 100644 --- a/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift +++ b/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift @@ -526,6 +526,15 @@ open class EPUBNavigatorViewController: UIViewController, VisualNavigator, Selec } } + /// Returns a `Locator` targeting the first visible HTML element on the current resource. + public func findFirstVisibleElementLocator(completion: @escaping (Locator?) -> Void) { + guard let spreadView = paginationView.currentView as? EPUBSpreadView else { + DispatchQueue.main.async { completion(nil) } + return + } + spreadView.findFirstVisibleElementLocator(completion: completion) + } + /// Last current location notified to the delegate. /// Used to avoid sending twice the same location. private var notifiedCurrentLocation: Locator? diff --git a/Sources/Navigator/EPUB/EPUBSpreadView.swift b/Sources/Navigator/EPUB/EPUBSpreadView.swift index c7b3f6405..7ae36009b 100644 --- a/Sources/Navigator/EPUB/EPUBSpreadView.swift +++ b/Sources/Navigator/EPUB/EPUBSpreadView.swift @@ -315,7 +315,23 @@ class EPUBSpreadView: UIView, Loggable, PageView { return false } - + func findFirstVisibleElementLocator(completion: @escaping (Locator?) -> Void) { + evaluateScript("readium.findFirstVisibleLocator()") { result in + DispatchQueue.main.async { + do { + let resource = self.spread.leading + let locator = try Locator(json: result.get)? + .copy(href: resource.href, type: resource.type ?? MediaType.xhtml.string) + completion(locator) + } catch { + self.log(.error, error) + completion(nil) + } + } + } + } + + // MARK: - JS Messages private var JSMessages: [String: (Any) -> Void] = [:] diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift new file mode 100644 index 000000000..a8f9e2645 --- /dev/null +++ b/Sources/Navigator/TTS/TTSController.swift @@ -0,0 +1,287 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import AVFoundation +import Foundation +import R2Shared + +public protocol TTSControllerDelegate: AnyObject { + func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) + + func ttsController(_ ttsController: TTSController, didStartSpeaking text: String, locale: String, at locator: Locator) + func ttsController(_ ttsController: TTSController, didStartSpeakingRangeAt locator: Locator) +} + +public extension TTSControllerDelegate { + func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) {} + func ttsController(_ ttsController: TTSController, didStartSpeaking text: String, locale: String, at locator: Locator) {} + func ttsController(_ ttsController: TTSController, didStartSpeakingRangeAt locator: Locator) {} +} + +public class TTSController { + + public struct Configuration { + public var defaultLanguage: String? + public var rate: Double + + public init(defaultLanguage: String? = nil, rate: Double = defaultRate) { + self.defaultLanguage = defaultLanguage + self.rate = rate + } + + public static let defaultRate: Double = Double(AVSpeechUtteranceDefaultSpeechRate) + public static let minimumRate: Double = Double(AVSpeechUtteranceMinimumSpeechRate) + public static let maximumRate: Double = Double(AVSpeechUtteranceMaximumSpeechRate) + } + + private let publication: Publication + public var config: Configuration + public weak var delegate: TTSControllerDelegate? + + private let queue = DispatchQueue.global(qos: .userInitiated) + + public init(publication: Publication, config: Configuration = Configuration(), delegate: TTSControllerDelegate? = nil) { + self.publication = publication + self.config = config + self.delegate = delegate + + adapter.controller = self + } + + public enum State { + case stopped, speaking, paused, failure(Error) + } + + public var state: State { + if let error = error { + return .failure(error) + } else { + return adapter.state + } + } + + private func notifyStateUpdate() { + delegate?.ttsController(self, stateDidChange: state) + } + + private var error: Error? { + didSet { + if oldValue != nil || error != nil { + notifyStateUpdate() + } + } + } + + public func playPause(from start: Locator? = nil) { + if case .speaking = state { + pause() + } else { + play(from: start) + } + } + + public func play(from start: Locator? = nil) { + if start == nil { + switch state { + case .stopped: + break + case .paused: + if adapter.continue() { + return + } + case .speaking: + return + case .failure: + break + } + } + + speakingUtteranceIndex = nil + utterances = [] + contentIterator = publication.contentIterator(from: start) + next() + } + + public func pause() { + adapter.pause() + } + + public func next() { + queue.async { [self] in + error = nil + do { + guard let utterance = try nextUtterance(direction: .forward) else { + return + } + adapter.speak(utterance) + } catch { + self.error = error + } + } + } + + // MARK: - Utterances + + private var contentIterator: ContentIterator? { + willSet { contentIterator?.close() } + } + + private struct Utterance { + let text: String + let locator: Locator + var language: String? = nil + var postDelay: TimeInterval? = nil + } + + private enum Direction { + case forward, backward + } + + private var speakingUtteranceIndex: Int? + private var utterances: [Utterance] = [] + + private func nextUtterance(direction: Direction) throws -> Utterance? { + guard let nextIndex = nextUtteranceIndex(direction: direction) else { + if try loadNextUtterances(direction: direction) { + return try nextUtterance(direction: direction) + } else { + return nil + } + } + speakingUtteranceIndex = nextIndex + return utterances[nextIndex] + } + + private func nextUtteranceIndex(direction: Direction) -> Int? { + let index: Int = { + switch direction { + case .forward: + return (speakingUtteranceIndex ?? -1) + 1 + case .backward: + return (speakingUtteranceIndex ?? utterances.count) - 1 + } + }() + guard utterances.indices.contains(index) else { + return nil + } + return index + } + + private func loadNextUtterances(direction: Direction) throws -> Bool { + speakingUtteranceIndex = nil + utterances = [] + + guard let content: Content = try { + switch direction { + case .forward: + return try contentIterator?.next() + case .backward: + return try contentIterator?.previous() + } + }() else { + return false + } + + utterances = utterances(from: content) + guard !utterances.isEmpty else { + return try loadNextUtterances(direction: direction) + } + + return true + } + + private func utterances(from content: Content) -> [Utterance] { + switch content.data { + case .audio(target: _): + return [] + + case .image(target: _, description: let description): + guard let description = description, !description.isEmpty else { + return [] + } + return [Utterance(text: description, locator: content.locator)] + + case .text(spans: let spans, style: let style): + return spans.enumerated().map { offset, span in + Utterance( + text: span.text, + locator: span.locator, + language: span.language, + postDelay: (offset == spans.count - 1) ? 0.4 : nil + ) + } + } + } + + // MARK: – Speech Synthesizer + + private let adapter = SpeechSynthesizerAdapter() + + private var defaultLanguage: String { + config.defaultLanguage + ?? publication.metadata.languages.first + ?? AVSpeechSynthesisVoice.currentLanguageCode() + } + + private class SpeechSynthesizerAdapter: NSObject, AVSpeechSynthesizerDelegate { + let synthesizer = AVSpeechSynthesizer() + weak var controller: TTSController? + + override init() { + super.init() + synthesizer.delegate = self + } + + var state: State { + if synthesizer.isPaused { + return .paused + } else if synthesizer.isSpeaking { + return .speaking + } else { + return .stopped + } + } + + func pause() { + synthesizer.pauseSpeaking(at: .word) + } + + func `continue`() -> Bool { + synthesizer.continueSpeaking() + } + + func speak(_ utterance: Utterance) { + guard let controller = controller else { + return + } + let avUtterance = AVSpeechUtterance(string: utterance.text) + avUtterance.rate = Float(controller.config.rate) + avUtterance.postUtteranceDelay = utterance.postDelay ?? 0 + avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language ?? controller.defaultLanguage) + synthesizer.speak(avUtterance) + } + + func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didStart utterance: AVSpeechUtterance) { + controller?.notifyStateUpdate() + } + + func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didPause utterance: AVSpeechUtterance) { + controller?.notifyStateUpdate() + } + + func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didContinue utterance: AVSpeechUtterance) { + controller?.notifyStateUpdate() + } + + func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) { + controller?.notifyStateUpdate() + } + + func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + controller?.next() + } + } +} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/ContentIterator.swift b/Sources/Shared/Publication/Services/Content/ContentIterator.swift index b049233df..6be5acd87 100644 --- a/Sources/Shared/Publication/Services/Content/ContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/ContentIterator.swift @@ -40,9 +40,9 @@ public struct Content: Equatable { } public struct TextSpan: Equatable { - let locator: Locator - let language: String? - let text: String + public let locator: Locator + public let language: String? + public let text: String } } From 0ff9ad26f2c05fe77bba2af0ea7541bbc9203c7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Tue, 3 May 2022 19:38:15 +0200 Subject: [PATCH 10/46] Improve `TTSController`, add the `TTSViewModel` --- .../EPUB/EPUBNavigatorViewController.swift | 3 +- Sources/Navigator/EPUB/EPUBSpreadView.swift | 2 +- Sources/Navigator/Navigator.swift | 19 ++-- Sources/Navigator/TTS/TTSController.swift | 89 ++++++++++++++----- .../Content/ContentIterationService.swift | 11 ++- .../Reader/Common/ReaderViewController.swift | 28 +++--- .../Reader/Common/TTS/TTSViewModel.swift | 52 +++++++++++ .../Reader/EPUB/EPUBViewController.swift | 12 +-- 8 files changed, 162 insertions(+), 54 deletions(-) create mode 100644 TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift diff --git a/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift b/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift index 47da33c95..c77a85596 100644 --- a/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift +++ b/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift @@ -526,8 +526,7 @@ open class EPUBNavigatorViewController: UIViewController, VisualNavigator, Selec } } - /// Returns a `Locator` targeting the first visible HTML element on the current resource. - public func findFirstVisibleElementLocator(completion: @escaping (Locator?) -> Void) { + public func findLocationOfFirstVisibleContent(completion: @escaping (Locator?) -> ()) { guard let spreadView = paginationView.currentView as? EPUBSpreadView else { DispatchQueue.main.async { completion(nil) } return diff --git a/Sources/Navigator/EPUB/EPUBSpreadView.swift b/Sources/Navigator/EPUB/EPUBSpreadView.swift index 7ae36009b..a677e376f 100644 --- a/Sources/Navigator/EPUB/EPUBSpreadView.swift +++ b/Sources/Navigator/EPUB/EPUBSpreadView.swift @@ -320,7 +320,7 @@ class EPUBSpreadView: UIView, Loggable, PageView { DispatchQueue.main.async { do { let resource = self.spread.leading - let locator = try Locator(json: result.get)? + let locator = try Locator(json: result.get())? .copy(href: resource.href, type: resource.type ?? MediaType.xhtml.string) completion(locator) } catch { diff --git a/Sources/Navigator/Navigator.swift b/Sources/Navigator/Navigator.swift index 1ce3f9fe7..7d7b62ad0 100644 --- a/Sources/Navigator/Navigator.swift +++ b/Sources/Navigator/Navigator.swift @@ -20,6 +20,9 @@ public protocol Navigator { /// Can be used to save a bookmark to the current position. var currentLocation: Locator? { get } + /// Returns a `Locator` targeting the first visible content element on the current resource. + func findLocationOfFirstVisibleContent(completion: @escaping (Locator?) -> Void) + /// Moves to the position in the publication correponding to the given `Locator`. /// - Parameter completion: Called when the transition is completed. /// - Returns: Whether the navigator is able to move to the locator. The completion block is only called if true was returned. @@ -47,29 +50,35 @@ public protocol Navigator { } public extension Navigator { - + + func findLocationOfFirstVisibleContent(completion: @escaping (Locator?) -> ()) { + DispatchQueue.main.async { + completion(nil) + } + } + /// Adds default values for the parameters. @discardableResult func go(to locator: Locator, animated: Bool = false, completion: @escaping () -> Void = {}) -> Bool { - return go(to: locator, animated: animated, completion: completion) + go(to: locator, animated: animated, completion: completion) } /// Adds default values for the parameters. @discardableResult func go(to link: Link, animated: Bool = false, completion: @escaping () -> Void = {}) -> Bool { - return go(to: link, animated: animated, completion: completion) + go(to: link, animated: animated, completion: completion) } /// Adds default values for the parameters. @discardableResult func goForward(animated: Bool = false, completion: @escaping () -> Void = {}) -> Bool { - return goForward(animated: animated, completion: completion) + goForward(animated: animated, completion: completion) } /// Adds default values for the parameters. @discardableResult func goBackward(animated: Bool = false, completion: @escaping () -> Void = {}) -> Bool { - return goBackward(animated: animated, completion: completion) + goBackward(animated: animated, completion: completion) } } diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index a8f9e2645..2d081fcf3 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -11,39 +11,59 @@ import R2Shared public protocol TTSControllerDelegate: AnyObject { func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) - func ttsController(_ ttsController: TTSController, didStartSpeaking text: String, locale: String, at locator: Locator) - func ttsController(_ ttsController: TTSController, didStartSpeakingRangeAt locator: Locator) + func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance) + func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance, rangeAt locator: Locator) } public extension TTSControllerDelegate { func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) {} - func ttsController(_ ttsController: TTSController, didStartSpeaking text: String, locale: String, at locator: Locator) {} - func ttsController(_ ttsController: TTSController, didStartSpeakingRangeAt locator: Locator) {} + func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance) {} + func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance, rangeAt locator: Locator) {} } public class TTSController { + public class func canSpeak(_ publication: Publication) -> Bool { + publication.isContentIterable + } + public struct Configuration { public var defaultLanguage: String? public var rate: Double + public var pitch: Double - public init(defaultLanguage: String? = nil, rate: Double = defaultRate) { + public init( + defaultLanguage: String? = nil, + rate: Double = defaultRate, + pitch: Double = defaultPitch + ) { self.defaultLanguage = defaultLanguage self.rate = rate + self.pitch = pitch } + public static let defaultPitch: Double = 1.0 public static let defaultRate: Double = Double(AVSpeechUtteranceDefaultSpeechRate) public static let minimumRate: Double = Double(AVSpeechUtteranceMinimumSpeechRate) public static let maximumRate: Double = Double(AVSpeechUtteranceMaximumSpeechRate) } + public struct Utterance { + public let text: String + public let locator: Locator + public let language: String? + public let postDelay: TimeInterval + } + private let publication: Publication public var config: Configuration public weak var delegate: TTSControllerDelegate? - private let queue = DispatchQueue.global(qos: .userInitiated) + private let queue: DispatchQueue = .global(qos: .userInitiated) public init(publication: Publication, config: Configuration = Configuration(), delegate: TTSControllerDelegate? = nil) { + precondition(publication.isContentIterable, "The Publication must be iterable to be used with TTSController") + self.publication = publication self.config = config self.delegate = delegate @@ -63,10 +83,6 @@ public class TTSController { } } - private func notifyStateUpdate() { - delegate?.ttsController(self, stateDidChange: state) - } - private var error: Error? { didSet { if oldValue != nil || error != nil { @@ -109,6 +125,10 @@ public class TTSController { adapter.pause() } + public func stop() { + adapter.stop() + } + public func next() { queue.async { [self] in error = nil @@ -116,26 +136,34 @@ public class TTSController { guard let utterance = try nextUtterance(direction: .forward) else { return } + if !utterance.text.contains(where: { $0.isLetter || $0.isNumber }) { + next() + return + } adapter.speak(utterance) + DispatchQueue.main.async { + delegate?.ttsController(self, didStartSpeaking: utterance) + } } catch { self.error = error } } } + private func notifyStateUpdate() { + delegate?.ttsController(self, stateDidChange: state) + } + + private func notifySpeakingRange(_ range: Range) { + + } + // MARK: - Utterances private var contentIterator: ContentIterator? { willSet { contentIterator?.close() } } - private struct Utterance { - let text: String - let locator: Locator - var language: String? = nil - var postDelay: TimeInterval? = nil - } - private enum Direction { case forward, backward } @@ -202,15 +230,20 @@ public class TTSController { guard let description = description, !description.isEmpty else { return [] } - return [Utterance(text: description, locator: content.locator)] - - case .text(spans: let spans, style: let style): + return [Utterance( + text: description, + locator: content.locator, + language: nil, + postDelay: 0 + )] + + case .text(spans: let spans, style: _): return spans.enumerated().map { offset, span in Utterance( text: span.text, locator: span.locator, language: span.language, - postDelay: (offset == spans.count - 1) ? 0.4 : nil + postDelay: (offset == spans.count - 1) ? 0.4 : 0 ) } } @@ -249,6 +282,10 @@ public class TTSController { synthesizer.pauseSpeaking(at: .word) } + func stop() { + synthesizer.stopSpeaking(at: .word) + } + func `continue`() -> Bool { synthesizer.continueSpeaking() } @@ -259,7 +296,8 @@ public class TTSController { } let avUtterance = AVSpeechUtterance(string: utterance.text) avUtterance.rate = Float(controller.config.rate) - avUtterance.postUtteranceDelay = utterance.postDelay ?? 0 + avUtterance.pitchMultiplier = Float(controller.config.pitch) + avUtterance.postUtteranceDelay = utterance.postDelay avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language ?? controller.defaultLanguage) synthesizer.speak(avUtterance) } @@ -283,5 +321,12 @@ public class TTSController { func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { controller?.next() } + + func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance: AVSpeechUtterance) { + guard let range = Range(characterRange) else { + return + } + controller?.notifySpeakingRange(range) + } } } \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/ContentIterationService.swift b/Sources/Shared/Publication/Services/Content/ContentIterationService.swift index 1c89dc392..0d5c441fb 100644 --- a/Sources/Shared/Publication/Services/Content/ContentIterationService.swift +++ b/Sources/Shared/Publication/Services/Content/ContentIterationService.swift @@ -13,9 +13,16 @@ public protocol ContentIterationService: PublicationService { } public extension Publication { + var isContentIterable: Bool { + contentIterationService != nil + } + func contentIterator(from start: Locator?) -> ContentIterator? { - findService(ContentIterationService.self)? - .iterator(from: start) + contentIterationService?.iterator(from: start) + } + + private var contentIterationService: ContentIterationService? { + findService(ContentIterationService.self) } } diff --git a/TestApp/Sources/Reader/Common/ReaderViewController.swift b/TestApp/Sources/Reader/Common/ReaderViewController.swift index ec1ab7548..66c2edd95 100644 --- a/TestApp/Sources/Reader/Common/ReaderViewController.swift +++ b/TestApp/Sources/Reader/Common/ReaderViewController.swift @@ -1,13 +1,7 @@ // -// ReaderViewController.swift -// r2-testapp-swift -// -// Created by Mickaël Menu on 07.03.19. -// -// Copyright 2019 European Digital Reading Lab. All rights reserved. -// Licensed to the Readium Foundation under one or more contributor license agreements. -// Use of this source code is governed by a BSD-style license which is detailed in the -// LICENSE file present in the project repository where this source code is maintained. +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. // import Combine @@ -33,10 +27,13 @@ class ReaderViewController: UIViewController, Loggable { private(set) var stackView: UIStackView! private lazy var positionLabel = UILabel() private var subscriptions = Set() - + private var searchViewModel: SearchViewModel? private var searchViewController: UIHostingController? - + + private let ttsViewModel: TTSViewModel? + private var ttsBarViewController: UIHostingController? + /// This regex matches any string with at least 2 consecutive letters (not limited to ASCII). /// It's used when evaluating whether to display the body of a noteref referrer as the note's title. /// I.e. a `*` or `1` would not be used as a title, but `on` or `好書` would. @@ -55,6 +52,7 @@ class ReaderViewController: UIViewController, Loggable { self.books = books self.bookmarks = bookmarks self.highlights = highlights + self.ttsViewModel = TTSViewModel(navigator: navigator, publication: publication) super.init(nibName: nil, bundle: nil) @@ -137,11 +135,14 @@ class ReaderViewController: UIViewController, Loggable { } // Bookmarks buttons.append(UIBarButtonItem(image: #imageLiteral(resourceName: "bookmark"), style: .plain, target: self, action: #selector(bookmarkCurrentPosition))) - // Search if publication._isSearchable { buttons.append(UIBarButtonItem(image: UIImage(systemName: "magnifyingglass"), style: .plain, target: self, action: #selector(showSearchUI))) } + // Text to speech + if let ttsViewModel = ttsViewModel { + buttons.append(UIBarButtonItem(image: UIImage(systemName: "speaker.wave.2.fill"), style: .plain, target: ttsViewModel, action: #selector(TTSViewModel.start))) + } return buttons } @@ -215,6 +216,7 @@ class ReaderViewController: UIViewController, Loggable { } // MARK: - Search + @objc func showSearchUI() { if searchViewModel == nil { searchViewModel = SearchViewModel(publication: publication) @@ -237,7 +239,7 @@ class ReaderViewController: UIViewController, Loggable { present(vc, animated: true, completion: nil) searchViewController = vc } - + // MARK: - Highlights private func addHighlightDecorationsObserverOnce() { diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift new file mode 100644 index 000000000..da447f92f --- /dev/null +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -0,0 +1,52 @@ +// +// Copyright 2021 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation +import R2Navigator +import R2Shared + +final class TTSViewModel: ObservableObject, Loggable { + + @Published private(set) var state: TTSController.State = .stopped + + private let navigator: Navigator + private let publication: Publication + private var ttsController: TTSController! + + init?(navigator: Navigator, publication: Publication) { + guard TTSController.canSpeak(publication) else { + return nil + } + self.navigator = navigator + self.publication = publication + self.ttsController = TTSController(publication: publication, delegate: self) + } + + @objc func start() { + guard case .stopped = state else { + return + } + + navigator.findLocationOfFirstVisibleContent { [self] locator in + ttsController.play(from: locator ?? navigator.currentLocation) + } + } +} + +extension TTSViewModel: TTSControllerDelegate { + + func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) { + self.state = state + } + + func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance) { + log(.warning, "START SPEAKING \(utterance.text)") + navigator.go(to: utterance.locator) + (navigator as? DecorableNavigator)?.apply(decorations: [ + Decoration(id: "tts", locator: utterance.locator, style: .highlight(tint: .red)) + ], in: "tts") + } +} \ No newline at end of file diff --git a/TestApp/Sources/Reader/EPUB/EPUBViewController.swift b/TestApp/Sources/Reader/EPUB/EPUBViewController.swift index 0c7b42af7..e81bd9c6b 100644 --- a/TestApp/Sources/Reader/EPUB/EPUBViewController.swift +++ b/TestApp/Sources/Reader/EPUB/EPUBViewController.swift @@ -1,13 +1,7 @@ // -// EPUBViewController.swift -// r2-testapp-swift -// -// Created by Alexandre Camilleri on 7/3/17. -// -// Copyright 2018 European Digital Reading Lab. All rights reserved. -// Licensed to the Readium Foundation under one or more contributor license agreements. -// Use of this source code is governed by a BSD-style license which is detailed in the -// LICENSE file present in the project repository where this source code is maintained. +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. // import UIKit From f346158f0e0f3dbc5efbe398c6a3e900213fdb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 5 May 2022 12:08:11 +0200 Subject: [PATCH 11/46] Add the TTS settings --- Sources/Navigator/TTS/TTSController.swift | 75 ++++++------ .../Navigator/Toolkit/Extensions/Range.swift | 20 ++++ .../Common/Toolkit/Extensions/String.swift | 15 +++ TestApp/Sources/Common/UX/IconButton.swift | 40 +++++++ .../Reader/Common/ReaderViewController.swift | 38 ++++++- .../Sources/Reader/Common/TTS/TTSView.swift | 107 ++++++++++++++++++ .../Reader/Common/TTS/TTSViewModel.swift | 50 ++++++-- 7 files changed, 300 insertions(+), 45 deletions(-) create mode 100644 Sources/Navigator/Toolkit/Extensions/Range.swift create mode 100644 TestApp/Sources/Common/UX/IconButton.swift create mode 100644 TestApp/Sources/Reader/Common/TTS/TTSView.swift diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index 2d081fcf3..428f43b82 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -19,9 +19,26 @@ public extension TTSControllerDelegate { func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) {} func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance) {} func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance, rangeAt locator: Locator) {} + func ttsController(_ ttsController: TTSController, didReceiveError error: Error) {} } -public class TTSController { +/// Range of valid values for an AVUtterance pitch. +/// +/// > Before enqueing the utterance, set this property to a value within the range of 0.5 for lower pitch to 2.0 for +/// > higher pitch. The default value is 1.0. +/// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier +private let avPitchRange = 0.5...2.0 + +/// Range of valid values for an AVUtterance rate. +/// +/// > The speech rate is a decimal representation within the range of `AVSpeechUtteranceMinimumSpeechRate` and +/// > `AVSpeechUtteranceMaximumSpeechRate`. Lower values correspond to slower speech, and higher values correspond to +/// > faster speech. The default value is `AVSpeechUtteranceDefaultSpeechRate`. +/// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619708-rate +private let avRateRange = + Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate) + +public class TTSController: Loggable { public class func canSpeak(_ publication: Publication) -> Bool { publication.isContentIterable @@ -42,10 +59,8 @@ public class TTSController { self.pitch = pitch } - public static let defaultPitch: Double = 1.0 - public static let defaultRate: Double = Double(AVSpeechUtteranceDefaultSpeechRate) - public static let minimumRate: Double = Double(AVSpeechUtteranceMinimumSpeechRate) - public static let maximumRate: Double = Double(AVSpeechUtteranceMaximumSpeechRate) + public static let defaultRate: Double = avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)) + public static let defaultPitch: Double = avPitchRange.percentageForValue(1.0) } public struct Utterance { @@ -71,28 +86,14 @@ public class TTSController { adapter.controller = self } - public enum State { - case stopped, speaking, paused, failure(Error) - } - - public var state: State { - if let error = error { - return .failure(error) - } else { - return adapter.state - } + public enum State: Equatable { + case stopped, speaking, paused } - private var error: Error? { - didSet { - if oldValue != nil || error != nil { - notifyStateUpdate() - } - } - } + public var state: State { adapter.state } public func playPause(from start: Locator? = nil) { - if case .speaking = state { + if state == .speaking { pause() } else { play(from: start) @@ -110,14 +111,13 @@ public class TTSController { } case .speaking: return - case .failure: - break } } speakingUtteranceIndex = nil utterances = [] contentIterator = publication.contentIterator(from: start) + next() } @@ -126,14 +126,18 @@ public class TTSController { } public func stop() { + speakingUtteranceIndex = nil + utterances = [] + contentIterator = nil + adapter.stop() } public func next() { queue.async { [self] in - error = nil do { guard let utterance = try nextUtterance(direction: .forward) else { + notifyStateUpdate() return } if !utterance.text.contains(where: { $0.isLetter || $0.isNumber }) { @@ -145,13 +149,18 @@ public class TTSController { delegate?.ttsController(self, didStartSpeaking: utterance) } } catch { - self.error = error + DispatchQueue.main.async { + notifyStateUpdate() + delegate?.ttsController(self, didReceiveError: error) + } } } } private func notifyStateUpdate() { - delegate?.ttsController(self, stateDidChange: state) + DispatchQueue.main.async { [self] in + delegate?.ttsController(self, stateDidChange: state) + } } private func notifySpeakingRange(_ range: Range) { @@ -279,11 +288,13 @@ public class TTSController { } func pause() { - synthesizer.pauseSpeaking(at: .word) + synthesizer.pauseSpeaking(at: .immediate) + controller?.notifyStateUpdate() } func stop() { - synthesizer.stopSpeaking(at: .word) + synthesizer.stopSpeaking(at: .immediate) + controller?.notifyStateUpdate() } func `continue`() -> Bool { @@ -295,8 +306,8 @@ public class TTSController { return } let avUtterance = AVSpeechUtterance(string: utterance.text) - avUtterance.rate = Float(controller.config.rate) - avUtterance.pitchMultiplier = Float(controller.config.pitch) + avUtterance.rate = Float(avRateRange.valueForPercentage(controller.config.rate)) + avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(controller.config.pitch)) avUtterance.postUtteranceDelay = utterance.postDelay avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language ?? controller.defaultLanguage) synthesizer.speak(avUtterance) diff --git a/Sources/Navigator/Toolkit/Extensions/Range.swift b/Sources/Navigator/Toolkit/Extensions/Range.swift new file mode 100644 index 000000000..658df929e --- /dev/null +++ b/Sources/Navigator/Toolkit/Extensions/Range.swift @@ -0,0 +1,20 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +extension ClosedRange where Bound == Double { + + /// Returns the equivalent percentage from 0.0 to 1.0 for the given `value` in the range. + func percentageForValue(_ value: Bound) -> Double { + (value - lowerBound) / (upperBound - lowerBound) + } + + /// Returns the actual value in the range for the given `percentage` from 0.0 to 1.0. + func valueForPercentage(_ percentage: Double) -> Bound { + ((upperBound - lowerBound) * percentage) + lowerBound + } +} diff --git a/TestApp/Sources/Common/Toolkit/Extensions/String.swift b/TestApp/Sources/Common/Toolkit/Extensions/String.swift index 6febdc9b9..9e89ad542 100644 --- a/TestApp/Sources/Common/Toolkit/Extensions/String.swift +++ b/TestApp/Sources/Common/Toolkit/Extensions/String.swift @@ -19,4 +19,19 @@ extension String { return components(separatedBy: invalidCharacters) .joined(separator: " ") } + + /// Formats a `percentage` into a localized String. + static func localizedPercentage(_ percentage: Double) -> String { + percentageFormatter.string(from: NSNumber(value: percentage)) + ?? String(format: "%.0f%%", percentage) + } } + +private let percentageFormatter: NumberFormatter = { + let formatter = NumberFormatter() + formatter.numberStyle = .percent + formatter.minimumIntegerDigits = 1 + formatter.maximumIntegerDigits = 3 + formatter.maximumFractionDigits = 0 + return formatter +}() diff --git a/TestApp/Sources/Common/UX/IconButton.swift b/TestApp/Sources/Common/UX/IconButton.swift new file mode 100644 index 000000000..65cf3b69a --- /dev/null +++ b/TestApp/Sources/Common/UX/IconButton.swift @@ -0,0 +1,40 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation +import SwiftUI + +struct IconButton: View { + enum Size: CGFloat { + case small = 24 + case medium = 32 + } + + private let systemName: String + private let foregroundColor: Color + private let size: Size + private let action: () -> Void + + init(systemName: String, foregroundColor: Color = Color(UIColor.label), size: Size = .medium, action: @escaping () -> Void) { + self.systemName = systemName + self.foregroundColor = foregroundColor + self.size = size + self.action = action + } + + var body: some View { + Button( + action: action, + label: { + Image(systemName: systemName) + .resizable() + .scaledToFit() + .foregroundColor(foregroundColor) + } + ) + .frame(width: size.rawValue, height: size.rawValue) + } +} diff --git a/TestApp/Sources/Reader/Common/ReaderViewController.swift b/TestApp/Sources/Reader/Common/ReaderViewController.swift index 66c2edd95..87c8bf2ad 100644 --- a/TestApp/Sources/Reader/Common/ReaderViewController.swift +++ b/TestApp/Sources/Reader/Common/ReaderViewController.swift @@ -32,7 +32,7 @@ class ReaderViewController: UIViewController, Loggable { private var searchViewController: UIHostingController? private let ttsViewModel: TTSViewModel? - private var ttsBarViewController: UIHostingController? + private let ttsControlsViewController: UIHostingController? /// This regex matches any string with at least 2 consecutive letters (not limited to ASCII). /// It's used when evaluating whether to display the body of a noteref referrer as the note's title. @@ -52,8 +52,15 @@ class ReaderViewController: UIViewController, Loggable { self.books = books self.bookmarks = bookmarks self.highlights = highlights - self.ttsViewModel = TTSViewModel(navigator: navigator, publication: publication) - + + if let model = TTSViewModel(navigator: navigator, publication: publication) { + ttsViewModel = model + ttsControlsViewController = UIHostingController(rootView: TTSControls(viewModel: model)) + } else { + ttsViewModel = nil + ttsControlsViewController = nil + } + super.init(nibName: nil, bundle: nil) addHighlightDecorationsObserverOnce() @@ -97,9 +104,9 @@ class ReaderViewController: UIViewController, Loggable { addChild(navigator) stackView.addArrangedSubview(navigator.view) navigator.didMove(toParent: self) - + stackView.addArrangedSubview(accessibilityToolbar) - + positionLabel.translatesAutoresizingMaskIntoConstraints = false positionLabel.font = .systemFont(ofSize: 12) positionLabel.textColor = .darkGray @@ -108,6 +115,25 @@ class ReaderViewController: UIViewController, Loggable { positionLabel.centerXAnchor.constraint(equalTo: view.centerXAnchor), positionLabel.bottomAnchor.constraint(equalTo: navigator.view.bottomAnchor, constant: -20) ]) + + if let state = ttsViewModel?.$state, let controls = ttsControlsViewController { + controls.view.backgroundColor = .clear + + addChild(controls) + controls.view.translatesAutoresizingMaskIntoConstraints = false + view.addSubview(controls.view) + NSLayoutConstraint.activate([ + controls.view.centerXAnchor.constraint(equalTo: view.centerXAnchor), + controls.view.topAnchor.constraint(equalTo: view.safeAreaLayoutGuide.topAnchor, constant: 20), + ]) + controls.didMove(toParent: self) + + state + .sink { [unowned self] state in + controls.view.isHidden = (state == .stopped) + } + .store(in: &subscriptions) + } } override func willMove(toParent parent: UIViewController?) { @@ -141,7 +167,7 @@ class ReaderViewController: UIViewController, Loggable { } // Text to speech if let ttsViewModel = ttsViewModel { - buttons.append(UIBarButtonItem(image: UIImage(systemName: "speaker.wave.2.fill"), style: .plain, target: ttsViewModel, action: #selector(TTSViewModel.start))) + buttons.append(UIBarButtonItem(image: UIImage(systemName: "speaker.wave.2.fill"), style: .plain, target: ttsViewModel, action: #selector(TTSViewModel.play))) } return buttons diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift new file mode 100644 index 000000000..f31830aeb --- /dev/null +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -0,0 +1,107 @@ +// +// Copyright 2021 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation +import R2Navigator +import SwiftUI + +struct TTSControls: View { + @ObservedObject var viewModel: TTSViewModel + @State private var showSettings = false + + var body: some View { + HStack( + alignment: .center, + spacing: 16 + ) { + + IconButton( + systemName: "backward.fill", + size: .small, + action: { showSettings.toggle() } + ) + + IconButton( + systemName: (viewModel.state == .speaking) ? "pause.fill" : "play.fill", + action: { viewModel.playPause() } + ) + + IconButton( + systemName: "stop.fill", + action: { viewModel.stop() } + ) + + IconButton( + systemName: "forward.fill", + size: .small, + action: { showSettings.toggle() } + ) + + Spacer(minLength: 0) + + IconButton( + systemName: "gearshape.fill", + size: .small, + action: { showSettings.toggle() } + ) + .popover(isPresented: $showSettings) { + TTSSettings(viewModel: viewModel) + .frame( + minWidth: 320, idealWidth: 400, maxWidth: nil, + minHeight: 150, idealHeight: 150, maxHeight: nil, + alignment: .top + ) + } + } + .padding(16) + .frame(height: 44) + .frame(maxWidth: .infinity) + .background(Color(UIColor.systemBackground)) + .opacity(0.8) + .cornerRadius(16) + } +} + +struct TTSSettings: View { + @ObservedObject var viewModel: TTSViewModel + + var body: some View { + List { + Section(header: Text("Speech settings")) { + ConfigStepper( + for: \.rate, + step: TTSController.Configuration.defaultRate / 10, + caption: "Rate" + ) + + ConfigStepper( + for: \.pitch, + step: TTSController.Configuration.defaultPitch / 4, + caption: "Pitch" + ) + } + } + .listStyle(.insetGrouped) + } + + @ViewBuilder func ConfigStepper( + for keyPath: WritableKeyPath, + step: Double, + caption: String + ) -> some View { + Stepper( + value: Binding( + get: { viewModel.config[keyPath: keyPath] }, + set: { viewModel.config[keyPath: keyPath] = $0 } + ), + in: 0.0...1.0, + step: step + ) { + Text(caption) + Text(String.localizedPercentage(viewModel.config[keyPath: keyPath])).font(.footnote) + } + } +} diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index da447f92f..eaa6742fa 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -4,6 +4,7 @@ // available in the top-level LICENSE file of the project. // +import Combine import Foundation import R2Navigator import R2Shared @@ -12,9 +13,12 @@ final class TTSViewModel: ObservableObject, Loggable { @Published private(set) var state: TTSController.State = .stopped + @Published var config: TTSController.Configuration = TTSController.Configuration() + private let navigator: Navigator private let publication: Publication private var ttsController: TTSController! + private var subscriptions: Set = [] init?(navigator: Navigator, publication: Publication) { guard TTSController.canSpeak(publication) else { @@ -22,11 +26,17 @@ final class TTSViewModel: ObservableObject, Loggable { } self.navigator = navigator self.publication = publication - self.ttsController = TTSController(publication: publication, delegate: self) + self.ttsController = TTSController(publication: publication, config: config, delegate: self) + + $config + .sink { [unowned self] in + ttsController.config = $0 + } + .store(in: &subscriptions) } - @objc func start() { - guard case .stopped = state else { + @objc func play() { + guard state == .stopped else { return } @@ -34,19 +44,45 @@ final class TTSViewModel: ObservableObject, Loggable { ttsController.play(from: locator ?? navigator.currentLocation) } } + + @objc func playPause() { + ttsController.playPause() + } + + @objc func stop() { + ttsController.stop() + } + + private func highlight(_ utterance: TTSController.Utterance?) { + guard let navigator = navigator as? DecorableNavigator else { + return + } + + var decorations: [Decoration] = [] + if let utterance = utterance { + decorations.append(Decoration( + id: "tts", + locator: utterance.locator, + style: .highlight(tint: .red) + )) + } + + navigator.apply(decorations: decorations, in: "tts") + } } extension TTSViewModel: TTSControllerDelegate { func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) { self.state = state + + if state == .stopped { + highlight(nil) + } } func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance) { - log(.warning, "START SPEAKING \(utterance.text)") navigator.go(to: utterance.locator) - (navigator as? DecorableNavigator)?.apply(decorations: [ - Decoration(id: "tts", locator: utterance.locator, style: .highlight(tint: .red)) - ], in: "tts") + highlight(utterance) } } \ No newline at end of file From f14d6135b9946004c240e93b8b48883f6b5b20dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 5 May 2022 18:40:54 +0200 Subject: [PATCH 12/46] Extract a `TTSEngine` protocol from the `TTSController` --- Sources/Navigator/TTS/TTSController.swift | 333 ++++++++++-------- .../Navigator/Toolkit/Extensions/Range.swift | 10 +- TestApp/Sources/App/AppModule.swift | 2 +- .../Reader/Common/ReaderViewController.swift | 11 +- .../Sources/Reader/Common/TTS/TTSView.swift | 16 +- .../Reader/Common/TTS/TTSViewModel.swift | 54 +-- 6 files changed, 236 insertions(+), 190 deletions(-) diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index 428f43b82..3bf4d7963 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -9,40 +9,30 @@ import Foundation import R2Shared public protocol TTSControllerDelegate: AnyObject { - func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) + func ttsController(_ ttsController: TTSController, playingDidChange isPlaying: Bool) - func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance) - func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance, rangeAt locator: Locator) + func ttsController(_ ttsController: TTSController, willStartSpeaking utterance: TTSUtterance) + func ttsController(_ ttsController: TTSController, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) + + func ttsController(_ ttsController: TTSController, didReceiveError error: Error) } public extension TTSControllerDelegate { - func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) {} - func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance) {} - func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance, rangeAt locator: Locator) {} - func ttsController(_ ttsController: TTSController, didReceiveError error: Error) {} + func ttsController(_ ttsController: TTSController, playingDidChange isPlaying: Bool) {} + func ttsController(_ ttsController: TTSController, willStartSpeaking utterance: TTSUtterance) {} + func ttsController(_ ttsController: TTSController, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) {} } -/// Range of valid values for an AVUtterance pitch. -/// -/// > Before enqueing the utterance, set this property to a value within the range of 0.5 for lower pitch to 2.0 for -/// > higher pitch. The default value is 1.0. -/// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier -private let avPitchRange = 0.5...2.0 - -/// Range of valid values for an AVUtterance rate. -/// -/// > The speech rate is a decimal representation within the range of `AVSpeechUtteranceMinimumSpeechRate` and -/// > `AVSpeechUtteranceMaximumSpeechRate`. Lower values correspond to slower speech, and higher values correspond to -/// > faster speech. The default value is `AVSpeechUtteranceDefaultSpeechRate`. -/// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619708-rate -private let avRateRange = - Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate) - -public class TTSController: Loggable { +public struct TTSUtterance: Equatable { + public let text: String + public let locator: Locator + public let language: String? + public let pitch: Double? + public let rate: Double? + public let postDelay: TimeInterval +} - public class func canSpeak(_ publication: Publication) -> Bool { - publication.isContentIterable - } +public class TTSController: Loggable, TTSEngineDelegate { public struct Configuration { public var defaultLanguage: String? @@ -50,50 +40,59 @@ public class TTSController: Loggable { public var pitch: Double public init( - defaultLanguage: String? = nil, - rate: Double = defaultRate, - pitch: Double = defaultPitch + defaultLanguage: String?, + rate: Double, + pitch: Double ) { self.defaultLanguage = defaultLanguage self.rate = rate self.pitch = pitch } - - public static let defaultRate: Double = avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)) - public static let defaultPitch: Double = avPitchRange.percentageForValue(1.0) } - public struct Utterance { - public let text: String - public let locator: Locator - public let language: String? - public let postDelay: TimeInterval + public class func canSpeak(_ publication: Publication) -> Bool { + publication.isContentIterable } - private let publication: Publication + public let defaultRate: Double + public let defaultPitch: Double public var config: Configuration public weak var delegate: TTSControllerDelegate? + private let publication: Publication + private let engine: TTSEngine private let queue: DispatchQueue = .global(qos: .userInitiated) - public init(publication: Publication, config: Configuration = Configuration(), delegate: TTSControllerDelegate? = nil) { + public init(publication: Publication, engine: TTSEngine = AVTTSEngine(), delegate: TTSControllerDelegate? = nil) { precondition(publication.isContentIterable, "The Publication must be iterable to be used with TTSController") - self.publication = publication - self.config = config + self.defaultRate = engine.defaultRate ?? 0.5 + self.defaultPitch = engine.defaultPitch ?? 0.5 + self.config = Configuration(defaultLanguage: nil, rate: defaultRate, pitch: defaultPitch) self.delegate = delegate + self.publication = publication + self.engine = engine - adapter.controller = self + engine.delegate = self } - public enum State: Equatable { - case stopped, speaking, paused + deinit { + engine.stop() + contentIterator?.close() } - public var state: State { adapter.state } + public var isPlaying: Bool = false { + didSet { + if oldValue != isPlaying { + DispatchQueue.main.async { [self] in + delegate?.ttsController(self, playingDidChange: isPlaying) + } + } + } + } public func playPause(from start: Locator? = nil) { - if state == .speaking { + if isPlaying { pause() } else { play(from: start) @@ -101,72 +100,57 @@ public class TTSController: Loggable { } public func play(from start: Locator? = nil) { - if start == nil { - switch state { - case .stopped: - break - case .paused: - if adapter.continue() { - return - } - case .speaking: - return - } + if start != nil { + speakingUtteranceIndex = nil + utterances = [] + contentIterator = publication.contentIterator(from: start) } - speakingUtteranceIndex = nil - utterances = [] - contentIterator = publication.contentIterator(from: start) + if contentIterator == nil { + contentIterator = publication.contentIterator(from: nil) + } - next() + if let utterance = currentUtterance { + play(utterance) + } else { + next() + } } - public func pause() { - adapter.pause() + private func play(_ utterance: TTSUtterance) { + DispatchQueue.main.async { [self] in + delegate?.ttsController(self, willStartSpeaking: utterance) + isPlaying = true + engine.speak(utterance) + } } - public func stop() { - speakingUtteranceIndex = nil - utterances = [] - contentIterator = nil - - adapter.stop() + public func pause() { + isPlaying = false + engine.stop() } public func next() { queue.async { [self] in do { guard let utterance = try nextUtterance(direction: .forward) else { - notifyStateUpdate() + isPlaying = false return } if !utterance.text.contains(where: { $0.isLetter || $0.isNumber }) { next() return } - adapter.speak(utterance) - DispatchQueue.main.async { - delegate?.ttsController(self, didStartSpeaking: utterance) - } + play(utterance) + } catch { DispatchQueue.main.async { - notifyStateUpdate() delegate?.ttsController(self, didReceiveError: error) } } } } - private func notifyStateUpdate() { - DispatchQueue.main.async { [self] in - delegate?.ttsController(self, stateDidChange: state) - } - } - - private func notifySpeakingRange(_ range: Range) { - - } - // MARK: - Utterances private var contentIterator: ContentIterator? { @@ -178,9 +162,13 @@ public class TTSController: Loggable { } private var speakingUtteranceIndex: Int? - private var utterances: [Utterance] = [] + private var utterances: [TTSUtterance] = [] + + private var currentUtterance: TTSUtterance? { + speakingUtteranceIndex.map { utterances[$0] } + } - private func nextUtterance(direction: Direction) throws -> Utterance? { + private func nextUtterance(direction: Direction) throws -> TTSUtterance? { guard let nextIndex = nextUtteranceIndex(direction: direction) else { if try loadNextUtterances(direction: direction) { return try nextUtterance(direction: direction) @@ -230,7 +218,7 @@ public class TTSController: Loggable { return true } - private func utterances(from content: Content) -> [Utterance] { + private func utterances(from content: Content) -> [TTSUtterance] { switch content.data { case .audio(target: _): return [] @@ -239,16 +227,11 @@ public class TTSController: Loggable { guard let description = description, !description.isEmpty else { return [] } - return [Utterance( - text: description, - locator: content.locator, - language: nil, - postDelay: 0 - )] + return [utterance(text: description, locator: content.locator)] case .text(spans: let spans, style: _): return spans.enumerated().map { offset, span in - Utterance( + utterance( text: span.text, locator: span.locator, language: span.language, @@ -258,9 +241,16 @@ public class TTSController: Loggable { } } - // MARK: – Speech Synthesizer - - private let adapter = SpeechSynthesizerAdapter() + private func utterance(text: String, locator: Locator, language: String? = nil, postDelay: TimeInterval = 0) -> TTSUtterance { + TTSUtterance( + text: text, + locator: locator, + language: language ?? defaultLanguage, + pitch: config.pitch, + rate: config.rate, + postDelay: postDelay + ) + } private var defaultLanguage: String { config.defaultLanguage @@ -268,76 +258,117 @@ public class TTSController: Loggable { ?? AVSpeechSynthesisVoice.currentLanguageCode() } - private class SpeechSynthesizerAdapter: NSObject, AVSpeechSynthesizerDelegate { - let synthesizer = AVSpeechSynthesizer() - weak var controller: TTSController? + // MARK: - TTSEngineDelegate - override init() { - super.init() - synthesizer.delegate = self + public func ttsEngine(_ engine: TTSEngine, didFinish utterance: TTSUtterance) { + if isPlaying && currentUtterance == utterance { + next() } + } - var state: State { - if synthesizer.isPaused { - return .paused - } else if synthesizer.isSpeaking { - return .speaking - } else { - return .stopped - } + public func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) { + DispatchQueue.main.async { [self] in + delegate?.ttsController(self, willSpeakRangeAt: locator, of: utterance) } + } +} - func pause() { - synthesizer.pauseSpeaking(at: .immediate) - controller?.notifyStateUpdate() - } +public protocol TTSEngineDelegate: AnyObject { + func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) + func ttsEngine(_ engine: TTSEngine, didFinish utterance: TTSUtterance) +} - func stop() { - synthesizer.stopSpeaking(at: .immediate) - controller?.notifyStateUpdate() - } +public protocol TTSEngine: AnyObject { + var defaultRate: Double? { get } + var defaultPitch: Double? { get } + var delegate: TTSEngineDelegate? { get set } - func `continue`() -> Bool { - synthesizer.continueSpeaking() - } + func speak(_ utterance: TTSUtterance) + func stop() +} - func speak(_ utterance: Utterance) { - guard let controller = controller else { - return - } - let avUtterance = AVSpeechUtterance(string: utterance.text) - avUtterance.rate = Float(avRateRange.valueForPercentage(controller.config.rate)) - avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(controller.config.pitch)) - avUtterance.postUtteranceDelay = utterance.postDelay - avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language ?? controller.defaultLanguage) - synthesizer.speak(avUtterance) - } +public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Loggable { - func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didStart utterance: AVSpeechUtterance) { - controller?.notifyStateUpdate() - } + public lazy var defaultRate: Double? = + avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)) + + /// Range of valid values for an AVUtterance rate. + /// + /// > The speech rate is a decimal representation within the range of `AVSpeechUtteranceMinimumSpeechRate` and + /// > `AVSpeechUtteranceMaximumSpeechRate`. Lower values correspond to slower speech, and higher values correspond to + /// > faster speech. The default value is `AVSpeechUtteranceDefaultSpeechRate`. + /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619708-rate + private let avRateRange = + Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate) + + public lazy var defaultPitch: Double? = + avPitchRange.percentageForValue(1.0) + + /// Range of valid values for an AVUtterance pitch. + /// + /// > Before enqueing the utterance, set this property to a value within the range of 0.5 for lower pitch to 2.0 for + /// > higher pitch. The default value is 1.0. + /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier + private let avPitchRange = 0.5...2.0 + + public weak var delegate: TTSEngineDelegate? + + private let synthesizer = AVSpeechSynthesizer() + + public override init() { + super.init() + synthesizer.delegate = self + } + + public func speak(_ utterance: TTSUtterance) { + synthesizer.stopSpeaking(at: .immediate) + synthesizer.speak(avUtterance(from: utterance)) + } - func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didPause utterance: AVSpeechUtterance) { - controller?.notifyStateUpdate() + public func stop() { + synthesizer.stopSpeaking(at: .immediate) + } + + private func avUtterance(from utterance: TTSUtterance) -> AVSpeechUtterance { + let avUtterance = AVUtterance(utterance: utterance) + if let rate = utterance.rate ?? defaultRate { + avUtterance.rate = Float(avRateRange.valueForPercentage(rate)) + } + if let pitch = utterance.pitch ?? defaultPitch { + avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(pitch)) } + avUtterance.postUtteranceDelay = utterance.postDelay + avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language) + return avUtterance + } - func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didContinue utterance: AVSpeechUtterance) { - controller?.notifyStateUpdate() + public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + guard let utterance = utterance as? AVUtterance else { + return } + delegate?.ttsEngine(self, didFinish: utterance.utterance) + } - func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) { - controller?.notifyStateUpdate() + public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance avUtterance: AVSpeechUtterance) { + guard + let delegate = delegate, + let range = Range(characterRange) + else { + return } +// controller?.notifySpeakingRange(range) + } - func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { - controller?.next() + private class AVUtterance: AVSpeechUtterance { + let utterance: TTSUtterance + + init(utterance: TTSUtterance) { + self.utterance = utterance + super.init(string: utterance.text) } - func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance: AVSpeechUtterance) { - guard let range = Range(characterRange) else { - return - } - controller?.notifySpeakingRange(range) + required init?(coder: NSCoder) { + fatalError("Not supported") } } } \ No newline at end of file diff --git a/Sources/Navigator/Toolkit/Extensions/Range.swift b/Sources/Navigator/Toolkit/Extensions/Range.swift index 658df929e..264df9101 100644 --- a/Sources/Navigator/Toolkit/Extensions/Range.swift +++ b/Sources/Navigator/Toolkit/Extensions/Range.swift @@ -6,15 +6,21 @@ import Foundation +extension ClosedRange { + func clamp(_ value: Bound) -> Bound { + Swift.min(Swift.max(lowerBound, value), upperBound) + } +} + extension ClosedRange where Bound == Double { /// Returns the equivalent percentage from 0.0 to 1.0 for the given `value` in the range. func percentageForValue(_ value: Bound) -> Double { - (value - lowerBound) / (upperBound - lowerBound) + return (clamp(value) - lowerBound) / (upperBound - lowerBound) } /// Returns the actual value in the range for the given `percentage` from 0.0 to 1.0. func valueForPercentage(_ percentage: Double) -> Bound { - ((upperBound - lowerBound) * percentage) + lowerBound + ((upperBound - lowerBound) * (0.0...1.0).clamp(percentage)) + lowerBound } } diff --git a/TestApp/Sources/App/AppModule.swift b/TestApp/Sources/App/AppModule.swift index ce2342fba..af69f1f7a 100644 --- a/TestApp/Sources/App/AppModule.swift +++ b/TestApp/Sources/App/AppModule.swift @@ -52,7 +52,7 @@ final class AppModule { opds = OPDSModule(delegate: self) // Set Readium 2's logging minimum level. - R2EnableLog(withMinimumSeverityLevel: .debug) + R2EnableLog(withMinimumSeverityLevel: .warning) } private(set) lazy var aboutViewController: UIViewController = { diff --git a/TestApp/Sources/Reader/Common/ReaderViewController.swift b/TestApp/Sources/Reader/Common/ReaderViewController.swift index 87c8bf2ad..1f6346c5b 100644 --- a/TestApp/Sources/Reader/Common/ReaderViewController.swift +++ b/TestApp/Sources/Reader/Common/ReaderViewController.swift @@ -53,16 +53,11 @@ class ReaderViewController: UIViewController, Loggable { self.bookmarks = bookmarks self.highlights = highlights - if let model = TTSViewModel(navigator: navigator, publication: publication) { - ttsViewModel = model - ttsControlsViewController = UIHostingController(rootView: TTSControls(viewModel: model)) - } else { - ttsViewModel = nil - ttsControlsViewController = nil - } + ttsViewModel = TTSViewModel(navigator: navigator, publication: publication) + ttsControlsViewController = ttsViewModel.map { UIHostingController(rootView: TTSControls(viewModel: $0)) } super.init(nibName: nil, bundle: nil) - + addHighlightDecorationsObserverOnce() updateHighlightDecorations() diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index f31830aeb..1a87fbb14 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -25,7 +25,7 @@ struct TTSControls: View { ) IconButton( - systemName: (viewModel.state == .speaking) ? "pause.fill" : "play.fill", + systemName: (viewModel.state == .playing) ? "pause.fill" : "play.fill", action: { viewModel.playPause() } ) @@ -37,7 +37,7 @@ struct TTSControls: View { IconButton( systemName: "forward.fill", size: .small, - action: { showSettings.toggle() } + action: { viewModel.next() } ) Spacer(minLength: 0) @@ -72,15 +72,15 @@ struct TTSSettings: View { List { Section(header: Text("Speech settings")) { ConfigStepper( + caption: "Rate", for: \.rate, - step: TTSController.Configuration.defaultRate / 10, - caption: "Rate" + step: viewModel.defaultRate / 10 ) ConfigStepper( + caption: "Pitch", for: \.pitch, - step: TTSController.Configuration.defaultPitch / 4, - caption: "Pitch" + step: viewModel.defaultPitch / 4 ) } } @@ -88,9 +88,9 @@ struct TTSSettings: View { } @ViewBuilder func ConfigStepper( + caption: String, for keyPath: WritableKeyPath, - step: Double, - caption: String + step: Double ) -> some View { Stepper( value: Binding( diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index eaa6742fa..76f1aa580 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -11,49 +11,60 @@ import R2Shared final class TTSViewModel: ObservableObject, Loggable { - @Published private(set) var state: TTSController.State = .stopped + enum State { + case stopped, paused, playing + } - @Published var config: TTSController.Configuration = TTSController.Configuration() + @Published private(set) var state: State = .stopped + @Published var config: TTSController.Configuration + private let tts: TTSController private let navigator: Navigator private let publication: Publication - private var ttsController: TTSController! private var subscriptions: Set = [] init?(navigator: Navigator, publication: Publication) { guard TTSController.canSpeak(publication) else { return nil } + self.tts = TTSController(publication: publication) + self.config = tts.config self.navigator = navigator self.publication = publication - self.ttsController = TTSController(publication: publication, config: config, delegate: self) + + tts.delegate = self $config .sink { [unowned self] in - ttsController.config = $0 + tts.config = $0 } .store(in: &subscriptions) } - @objc func play() { - guard state == .stopped else { - return - } + var defaultRate: Double { tts.defaultRate } + var defaultPitch: Double { tts.defaultPitch } + @objc func play() { navigator.findLocationOfFirstVisibleContent { [self] locator in - ttsController.play(from: locator ?? navigator.currentLocation) + tts.play(from: locator ?? navigator.currentLocation) } } @objc func playPause() { - ttsController.playPause() + tts.playPause() } @objc func stop() { - ttsController.stop() + state = .stopped + highlight(nil) + tts.pause() } - private func highlight(_ utterance: TTSController.Utterance?) { + @objc func next() { + tts.next() + } + + private func highlight(_ utterance: TTSUtterance?) { guard let navigator = navigator as? DecorableNavigator else { return } @@ -72,16 +83,19 @@ final class TTSViewModel: ObservableObject, Loggable { } extension TTSViewModel: TTSControllerDelegate { - - func ttsController(_ ttsController: TTSController, stateDidChange state: TTSController.State) { - self.state = state - - if state == .stopped { - highlight(nil) + public func ttsController(_ ttsController: TTSController, playingDidChange isPlaying: Bool) { + if isPlaying { + state = .playing + } else if state != .stopped { + state = .paused } } - func ttsController(_ ttsController: TTSController, didStartSpeaking utterance: TTSController.Utterance) { + public func ttsController(_ ttsController: TTSController, didReceiveError error: Error) { + log(.error, error) + } + + public func ttsController(_ ttsController: TTSController, willStartSpeaking utterance: TTSUtterance) { navigator.go(to: utterance.locator) highlight(utterance) } From 1be7114fcc7ccc5c5427df79ccb7fa0fc31924d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 5 May 2022 19:04:18 +0200 Subject: [PATCH 13/46] Refactor `TTSController` --- Sources/Navigator/TTS/TTSController.swift | 194 +++++------------- Sources/Navigator/TTS/TTSEngine.swift | 114 ++++++++++ .../Reader/Common/TTS/TTSViewModel.swift | 2 +- 3 files changed, 165 insertions(+), 145 deletions(-) create mode 100644 Sources/Navigator/TTS/TTSEngine.swift diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index 3bf4d7963..97bead63e 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -4,7 +4,6 @@ // available in the top-level LICENSE file of the project. // -import AVFoundation import Foundation import R2Shared @@ -50,7 +49,7 @@ public class TTSController: Loggable, TTSEngineDelegate { } } - public class func canSpeak(_ publication: Publication) -> Bool { + public static func canPlay(_ publication: Publication) -> Bool { publication.isContentIterable } @@ -83,10 +82,9 @@ public class TTSController: Loggable, TTSEngineDelegate { public var isPlaying: Bool = false { didSet { + precondition(Thread.isMainThread, "TTSController.isPlaying must be mutated from the main thread") if oldValue != isPlaying { - DispatchQueue.main.async { [self] in - delegate?.ttsController(self, playingDidChange: isPlaying) - } + delegate?.ttsController(self, playingDidChange: isPlaying) } } } @@ -99,6 +97,12 @@ public class TTSController: Loggable, TTSEngineDelegate { } } + public func pause() { + precondition(Thread.isMainThread, "TTSController.pause() must be called from the main thread") + isPlaying = false + engine.stop() + } + public func play(from start: Locator? = nil) { if start != nil { speakingUtteranceIndex = nil @@ -117,32 +121,40 @@ public class TTSController: Loggable, TTSEngineDelegate { } } - private func play(_ utterance: TTSUtterance) { - DispatchQueue.main.async { [self] in - delegate?.ttsController(self, willStartSpeaking: utterance) - isPlaying = true - engine.speak(utterance) - } + public func previous() { + playNextUtterance(direction: .backward) } - public func pause() { - isPlaying = false - engine.stop() + public func next() { + playNextUtterance(direction: .forward) } - public func next() { + private enum Direction { + case forward, backward + } + + private var contentIterator: ContentIterator? { + willSet { contentIterator?.close() } + } + + private var speakingUtteranceIndex: Int? + private var utterances: [TTSUtterance] = [] + + private var currentUtterance: TTSUtterance? { + speakingUtteranceIndex.map { utterances[$0] } + } + + private func playNextUtterance(direction: Direction) { queue.async { [self] in do { - guard let utterance = try nextUtterance(direction: .forward) else { - isPlaying = false - return - } - if !utterance.text.contains(where: { $0.isLetter || $0.isNumber }) { - next() - return + let utterance = try nextUtterance(direction: direction) + DispatchQueue.main.async { + if let utterance = utterance { + play(utterance) + } else { + isPlaying = false + } } - play(utterance) - } catch { DispatchQueue.main.async { delegate?.ttsController(self, didReceiveError: error) @@ -151,21 +163,12 @@ public class TTSController: Loggable, TTSEngineDelegate { } } - // MARK: - Utterances - - private var contentIterator: ContentIterator? { - willSet { contentIterator?.close() } - } - - private enum Direction { - case forward, backward - } - - private var speakingUtteranceIndex: Int? - private var utterances: [TTSUtterance] = [] + private func play(_ utterance: TTSUtterance) { + precondition(Thread.isMainThread, "TTSController.play() must be called from the main thread") - private var currentUtterance: TTSUtterance? { - speakingUtteranceIndex.map { utterances[$0] } + delegate?.ttsController(self, willStartSpeaking: utterance) + isPlaying = true + engine.speak(utterance) } private func nextUtterance(direction: Direction) throws -> TTSUtterance? { @@ -227,10 +230,10 @@ public class TTSController: Loggable, TTSEngineDelegate { guard let description = description, !description.isEmpty else { return [] } - return [utterance(text: description, locator: content.locator)] + return Array(ofNotNil: utterance(text: description, locator: content.locator)) case .text(spans: let spans, style: _): - return spans.enumerated().map { offset, span in + return spans.enumerated().compactMap { offset, span in utterance( text: span.text, locator: span.locator, @@ -241,8 +244,11 @@ public class TTSController: Loggable, TTSEngineDelegate { } } - private func utterance(text: String, locator: Locator, language: String? = nil, postDelay: TimeInterval = 0) -> TTSUtterance { - TTSUtterance( + private func utterance(text: String, locator: Locator, language: String? = nil, postDelay: TimeInterval = 0) -> TTSUtterance? { + guard text.contains(where: { $0.isLetter || $0.isNumber }) else { + return nil + } + return TTSUtterance( text: text, locator: locator, language: language ?? defaultLanguage, @@ -252,10 +258,10 @@ public class TTSController: Loggable, TTSEngineDelegate { ) } - private var defaultLanguage: String { + private var defaultLanguage: String? { config.defaultLanguage ?? publication.metadata.languages.first - ?? AVSpeechSynthesisVoice.currentLanguageCode() + ?? engine.defaultLanguage } // MARK: - TTSEngineDelegate @@ -271,104 +277,4 @@ public class TTSController: Loggable, TTSEngineDelegate { delegate?.ttsController(self, willSpeakRangeAt: locator, of: utterance) } } -} - -public protocol TTSEngineDelegate: AnyObject { - func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) - func ttsEngine(_ engine: TTSEngine, didFinish utterance: TTSUtterance) -} - -public protocol TTSEngine: AnyObject { - var defaultRate: Double? { get } - var defaultPitch: Double? { get } - var delegate: TTSEngineDelegate? { get set } - - func speak(_ utterance: TTSUtterance) - func stop() -} - -public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Loggable { - - public lazy var defaultRate: Double? = - avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)) - - /// Range of valid values for an AVUtterance rate. - /// - /// > The speech rate is a decimal representation within the range of `AVSpeechUtteranceMinimumSpeechRate` and - /// > `AVSpeechUtteranceMaximumSpeechRate`. Lower values correspond to slower speech, and higher values correspond to - /// > faster speech. The default value is `AVSpeechUtteranceDefaultSpeechRate`. - /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619708-rate - private let avRateRange = - Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate) - - public lazy var defaultPitch: Double? = - avPitchRange.percentageForValue(1.0) - - /// Range of valid values for an AVUtterance pitch. - /// - /// > Before enqueing the utterance, set this property to a value within the range of 0.5 for lower pitch to 2.0 for - /// > higher pitch. The default value is 1.0. - /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier - private let avPitchRange = 0.5...2.0 - - public weak var delegate: TTSEngineDelegate? - - private let synthesizer = AVSpeechSynthesizer() - - public override init() { - super.init() - synthesizer.delegate = self - } - - public func speak(_ utterance: TTSUtterance) { - synthesizer.stopSpeaking(at: .immediate) - synthesizer.speak(avUtterance(from: utterance)) - } - - public func stop() { - synthesizer.stopSpeaking(at: .immediate) - } - - private func avUtterance(from utterance: TTSUtterance) -> AVSpeechUtterance { - let avUtterance = AVUtterance(utterance: utterance) - if let rate = utterance.rate ?? defaultRate { - avUtterance.rate = Float(avRateRange.valueForPercentage(rate)) - } - if let pitch = utterance.pitch ?? defaultPitch { - avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(pitch)) - } - avUtterance.postUtteranceDelay = utterance.postDelay - avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language) - return avUtterance - } - - public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { - guard let utterance = utterance as? AVUtterance else { - return - } - delegate?.ttsEngine(self, didFinish: utterance.utterance) - } - - public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance avUtterance: AVSpeechUtterance) { - guard - let delegate = delegate, - let range = Range(characterRange) - else { - return - } -// controller?.notifySpeakingRange(range) - } - - private class AVUtterance: AVSpeechUtterance { - let utterance: TTSUtterance - - init(utterance: TTSUtterance) { - self.utterance = utterance - super.init(string: utterance.text) - } - - required init?(coder: NSCoder) { - fatalError("Not supported") - } - } } \ No newline at end of file diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift new file mode 100644 index 000000000..d90f2a776 --- /dev/null +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -0,0 +1,114 @@ +// +// Copyright 2021 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import AVFoundation +import Foundation +import R2Shared + +public protocol TTSEngineDelegate: AnyObject { + func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) + func ttsEngine(_ engine: TTSEngine, didFinish utterance: TTSUtterance) +} + +public protocol TTSEngine: AnyObject { + var defaultLanguage: String? { get } + var defaultRate: Double? { get } + var defaultPitch: Double? { get } + var delegate: TTSEngineDelegate? { get set } + + func speak(_ utterance: TTSUtterance) + func stop() +} + +public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Loggable { + + public var defaultLanguage: String? { + AVSpeechSynthesisVoice.currentLanguageCode() + } + + public lazy var defaultRate: Double? = + avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)) + + /// Range of valid values for an AVUtterance rate. + /// + /// > The speech rate is a decimal representation within the range of `AVSpeechUtteranceMinimumSpeechRate` and + /// > `AVSpeechUtteranceMaximumSpeechRate`. Lower values correspond to slower speech, and higher values correspond to + /// > faster speech. The default value is `AVSpeechUtteranceDefaultSpeechRate`. + /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619708-rate + private let avRateRange = + Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate) + + public lazy var defaultPitch: Double? = + avPitchRange.percentageForValue(1.0) + + /// Range of valid values for an AVUtterance pitch. + /// + /// > Before enqueing the utterance, set this property to a value within the range of 0.5 for lower pitch to 2.0 for + /// > higher pitch. The default value is 1.0. + /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier + private let avPitchRange = 0.5...2.0 + + public weak var delegate: TTSEngineDelegate? + + private let synthesizer = AVSpeechSynthesizer() + + public override init() { + super.init() + synthesizer.delegate = self + } + + public func speak(_ utterance: TTSUtterance) { + synthesizer.stopSpeaking(at: .immediate) + synthesizer.speak(avUtterance(from: utterance)) + } + + public func stop() { + synthesizer.stopSpeaking(at: .immediate) + } + + private func avUtterance(from utterance: TTSUtterance) -> AVSpeechUtterance { + let avUtterance = AVUtterance(utterance: utterance) + if let rate = utterance.rate ?? defaultRate { + avUtterance.rate = Float(avRateRange.valueForPercentage(rate)) + } + if let pitch = utterance.pitch ?? defaultPitch { + avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(pitch)) + } + avUtterance.postUtteranceDelay = utterance.postDelay + avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language) + return avUtterance + } + + public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + guard let utterance = utterance as? AVUtterance else { + return + } + delegate?.ttsEngine(self, didFinish: utterance.utterance) + } + + public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance avUtterance: AVSpeechUtterance) { + guard + let delegate = delegate, + let range = Range(characterRange) + else { + return + } +// controller?.notifySpeakingRange(range) + } + + private class AVUtterance: AVSpeechUtterance { + let utterance: TTSUtterance + + init(utterance: TTSUtterance) { + self.utterance = utterance + super.init(string: utterance.text) + } + + required init?(coder: NSCoder) { + fatalError("Not supported") + } + } +} diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index 76f1aa580..5a042b0b2 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -24,7 +24,7 @@ final class TTSViewModel: ObservableObject, Loggable { private var subscriptions: Set = [] init?(navigator: Navigator, publication: Publication) { - guard TTSController.canSpeak(publication) else { + guard TTSController.canPlay(publication) else { return nil } self.tts = TTSController(publication: publication) From bfc92a59bc4a14e6f72217f568edb3dd9a5e2d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 9 May 2022 12:32:50 +0200 Subject: [PATCH 14/46] Implement previous() --- .../Services/Content/PublicationContentIterator.swift | 9 ++++++++- TestApp/Sources/Reader/Common/TTS/TTSView.swift | 2 +- TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift index 01c5d558b..8e5aed100 100644 --- a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift @@ -43,7 +43,14 @@ public class PublicationContentIterator: ContentIterator, Loggable { } public func previous() throws -> Content? { - nil + guard let iterator = iterator(by: -1) else { + return nil + } + guard let content = try iterator.previous() else { + currentIterator = nil + return try previous() + } + return content } public func next() throws -> Content? { diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index 1a87fbb14..e23c1b702 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -21,7 +21,7 @@ struct TTSControls: View { IconButton( systemName: "backward.fill", size: .small, - action: { showSettings.toggle() } + action: { viewModel.previous() } ) IconButton( diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index 5a042b0b2..eddb6b396 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -60,6 +60,10 @@ final class TTSViewModel: ObservableObject, Loggable { tts.pause() } + @objc func previous() { + tts.previous() + } + @objc func next() { tts.next() } From 53e7d1cac6f8a36903ec00c5fe8ecddd2184115d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 9 May 2022 15:43:52 +0200 Subject: [PATCH 15/46] Add various text tokenizers --- Sources/Shared/Toolkit/Extensions/Range.swift | 42 +++++++++++ .../Toolkit/Tokenizer/NLTokenizer.swift | 47 ++++++++++++ .../Toolkit/Tokenizer/NSTokenizer.swift | 71 +++++++++++++++++++ .../Toolkit/Tokenizer/SimpleTokenizer.swift | 45 ++++++++++++ .../Shared/Toolkit/Tokenizer/Tokenizer.swift | 38 ++++++++++ .../Toolkit/Tokenizer/NLTokenizerTests.swift | 62 ++++++++++++++++ .../Toolkit/Tokenizer/NSTokenizerTests.swift | 64 +++++++++++++++++ .../Tokenizer/SimpleTokenizerTests.swift | 63 ++++++++++++++++ 8 files changed, 432 insertions(+) create mode 100644 Sources/Shared/Toolkit/Extensions/Range.swift create mode 100644 Sources/Shared/Toolkit/Tokenizer/NLTokenizer.swift create mode 100644 Sources/Shared/Toolkit/Tokenizer/NSTokenizer.swift create mode 100644 Sources/Shared/Toolkit/Tokenizer/SimpleTokenizer.swift create mode 100644 Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift create mode 100644 Tests/SharedTests/Toolkit/Tokenizer/NLTokenizerTests.swift create mode 100644 Tests/SharedTests/Toolkit/Tokenizer/NSTokenizerTests.swift create mode 100644 Tests/SharedTests/Toolkit/Tokenizer/SimpleTokenizerTests.swift diff --git a/Sources/Shared/Toolkit/Extensions/Range.swift b/Sources/Shared/Toolkit/Extensions/Range.swift new file mode 100644 index 000000000..e54f08893 --- /dev/null +++ b/Sources/Shared/Toolkit/Extensions/Range.swift @@ -0,0 +1,42 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +extension Range where Bound == String.Index { + + /// Trims leading and trailing whitespaces and newlines from this range in the given `string`. + func trimmingWhitespaces(in string: String) -> Self { + var onlyWhitespaces = true + var range = self + + for i in string[range].indices.reversed() { + let char = string[i] + if char.isWhitespace || char.isNewline { + continue + } + onlyWhitespaces = false + range = range.lowerBound.. [Range] { + let tokenizer = NaturalLanguage.NLTokenizer(unit: unit) + tokenizer.string = text + if let language = language { + tokenizer.setLanguage(language) + } + + return tokenizer.tokens(for: text.startIndex.. 0 } + } +} + +private extension TokenUnit { + @available(iOS 12.0, *) + var nlUnit: NLTokenUnit { + switch self { + case .word: + return .word + case .sentence: + return .sentence + case .paragraph: + return .paragraph + } + } +} \ No newline at end of file diff --git a/Sources/Shared/Toolkit/Tokenizer/NSTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/NSTokenizer.swift new file mode 100644 index 000000000..0551b7150 --- /dev/null +++ b/Sources/Shared/Toolkit/Tokenizer/NSTokenizer.swift @@ -0,0 +1,71 @@ +// +// Copyright 2021 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +public enum NSTokenizerError: Error { + case rangeConversionFailed(range: NSRange, string: String) +} + +/// A text `Tokenizer` using iOS 11+'s `NSLinguisticTaggerUnit`. +/// +/// Prefer using NLTokenizer on iOS 12+. +@available(iOS 11.0, *) +public class NSTokenizer: Tokenizer, Loggable { + private let unit: NSLinguisticTaggerUnit + private let options: NSLinguisticTagger.Options + + public init( + unit: TokenUnit, + options: NSLinguisticTagger.Options = [.joinNames, .omitPunctuation, .omitWhitespace] + ) { + self.unit = unit.nsUnit + self.options = options + } + + public func tokenize(text: String) throws -> [Range] { + let tagger = NSLinguisticTagger(tagSchemes: [.tokenType], options: 0) + tagger.string = text + + var error: Error? + var tokens: [Range] = [] + tagger.enumerateTags( + in: NSRange(location: 0, length: text.utf16.count), + unit: unit, + scheme: .tokenType, + options: options + ) { _, nsRange, _ in + guard let range = Range(nsRange, in: text) else { + error = NSTokenizerError.rangeConversionFailed(range: nsRange, string: text) + return + } + tokens.append(range) + } + + if let error = error { + throw error + } + + return tokens + .map { $0.trimmingWhitespaces(in: text) } + // Remove empty ranges. + .filter { $0.upperBound.utf16Offset(in: text) - $0.lowerBound.utf16Offset(in: text) > 0 } + } +} + +private extension TokenUnit { + @available(iOS 11.0, *) + var nsUnit: NSLinguisticTaggerUnit { + switch self { + case .word: + return .word + case .sentence: + return .sentence + case .paragraph: + return .paragraph + } + } +} diff --git a/Sources/Shared/Toolkit/Tokenizer/SimpleTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/SimpleTokenizer.swift new file mode 100644 index 000000000..ff1b8addf --- /dev/null +++ b/Sources/Shared/Toolkit/Tokenizer/SimpleTokenizer.swift @@ -0,0 +1,45 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// A `Tokenizer` using the basic `NSString.enumerateSubstrings()` API. +/// +/// Prefer using `NLTokenizer` or `NSTokenizer` on more recent versions of iOS. +public class SimpleTokenizer: Tokenizer { + private let options: NSString.EnumerationOptions + + public init(unit: TokenUnit) { + self.options = unit.enumerationOptions.union(.substringNotRequired) + } + + public func tokenize(text: String) throws -> [Range] { + var tokens: [Range] = [] + text.enumerateSubstrings( + in: text.startIndex.. 0 } + } +} + +private extension TokenUnit { + var enumerationOptions: NSString.EnumerationOptions { + switch self { + case .word: + return .byWords + case .sentence: + return .bySentences + case .paragraph: + return .byParagraphs + } + } +} \ No newline at end of file diff --git a/Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift new file mode 100644 index 000000000..8bb2be91b --- /dev/null +++ b/Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift @@ -0,0 +1,38 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// A tokenizer splits a String into tokens (e.g. words, sentences, etc.). +public protocol Tokenizer { + + /// Splits the given `text` into tokens and return their range in the string. + func tokenize(text: String) throws -> [Range] +} + +/// A text token unit which can be used with a `Tokenizer`. +public enum TokenUnit { + case word, sentence, paragraph +} + +/// A default cluster `Tokenizer` taking advantage of the best capabilities of each iOS version. +public class DefaultTokenizer: Tokenizer { + private let tokenizer: Tokenizer + + public init(unit: TokenUnit, language: String? = nil) { + if #available(iOS 12.0, *) { + tokenizer = NLTokenizer(unit: unit, language: language) + } else if #available(iOS 11.0, *) { + tokenizer = NSTokenizer(unit: unit) + } else { + tokenizer = SimpleTokenizer(unit: unit) + } + } + + public func tokenize(text: String) throws -> [Range] { + try tokenizer.tokenize(text: text) + } +} \ No newline at end of file diff --git a/Tests/SharedTests/Toolkit/Tokenizer/NLTokenizerTests.swift b/Tests/SharedTests/Toolkit/Tokenizer/NLTokenizerTests.swift new file mode 100644 index 000000000..995b35d0e --- /dev/null +++ b/Tests/SharedTests/Toolkit/Tokenizer/NLTokenizerTests.swift @@ -0,0 +1,62 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import XCTest +@testable import R2Shared + +@available(iOS 12.0, *) +class NLTokenizerTests: XCTestCase { + + func testTokenizeEmptyText() { + let tokenizer = NLTokenizer(unit: .word) + XCTAssertEqual(try tokenizer.tokenize(text: ""), []) + } + + func testTokenizeByWords() { + let tokenizer = NLTokenizer(unit: .word) + let text = "He said: \n\"What?\"" + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + ["He", "said", "What"] + ) + } + + func testTokenizeBySentences() { + let tokenizer = NLTokenizer(unit: .sentence) + let text = + """ + Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble + In the end, she went ahead. + """ + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + [ + "Mr. Bougee said, looking above: \"and what is the use of a book?\".", + "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", + "In the end, she went ahead." + ] + ) + } + + func testTokenizeByParagraphs() { + let tokenizer = NLTokenizer(unit: .paragraph) + let text = + """ + Oh dear, what nonsense I'm talking! Really? + + Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. + Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. + """ + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + [ + "Oh dear, what nonsense I'm talking! Really?", + "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", + "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." + ] + ) + } +} diff --git a/Tests/SharedTests/Toolkit/Tokenizer/NSTokenizerTests.swift b/Tests/SharedTests/Toolkit/Tokenizer/NSTokenizerTests.swift new file mode 100644 index 000000000..7a86cb6b4 --- /dev/null +++ b/Tests/SharedTests/Toolkit/Tokenizer/NSTokenizerTests.swift @@ -0,0 +1,64 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import XCTest +@testable import R2Shared + +import NaturalLanguage + +@available(iOS 12.0, *) +class NSTokenizerTests: XCTestCase { + + func testTokenizeEmptyText() { + let tokenizer = NSTokenizer(unit: .word) + XCTAssertEqual(try tokenizer.tokenize(text: ""), []) + } + + func testTokenizeByWords() { + let tokenizer = NSTokenizer(unit: .word) + let text = "He said: \n\"What?\"" + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + ["He", "said", "What"] + ) + } + + func testTokenizeBySentences() { + let tokenizer = NSTokenizer(unit: .sentence) + let text = + """ + Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble + In the end, she went ahead. + """ + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + [ + "Mr. Bougee said, looking above: \"and what is the use of a book?\".", + "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", + "In the end, she went ahead." + ] + ) + } + + func testTokenizeByParagraphs() { + let tokenizer = NSTokenizer(unit: .paragraph) + let text = + """ + Oh dear, what nonsense I'm talking! Really? + + Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. + Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. + """ + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + [ + "Oh dear, what nonsense I'm talking! Really?", + "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", + "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." + ] + ) + } +} diff --git a/Tests/SharedTests/Toolkit/Tokenizer/SimpleTokenizerTests.swift b/Tests/SharedTests/Toolkit/Tokenizer/SimpleTokenizerTests.swift new file mode 100644 index 000000000..819d73895 --- /dev/null +++ b/Tests/SharedTests/Toolkit/Tokenizer/SimpleTokenizerTests.swift @@ -0,0 +1,63 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import XCTest +@testable import R2Shared + +@available(iOS 12.0, *) +class SimpleTokenizerTests: XCTestCase { + + func testTokenizeEmptyText() { + let tokenizer = SimpleTokenizer(unit: .word) + XCTAssertEqual(try tokenizer.tokenize(text: ""), []) + } + + func testTokenizeByWords() { + let tokenizer = SimpleTokenizer(unit: .word) + let text = "He said: \n\"What?\"" + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + ["He", "said", "What"] + ) + } + + func testTokenizeBySentences() { + let tokenizer = SimpleTokenizer(unit: .sentence) + let text = + """ + Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble + In the end, she went ahead. + """ + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + [ + "Mr.", + "Bougee said, looking above: \"and what is the use of a book?\".", + "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", + "In the end, she went ahead." + ] + ) + } + + func testTokenizeByParagraphs() { + let tokenizer = SimpleTokenizer(unit: .paragraph) + let text = + """ + Oh dear, what nonsense I'm talking! Really? + + Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. + Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. + """ + XCTAssertEqual( + try tokenizer.tokenize(text: text).map { String(text[$0]) }, + [ + "Oh dear, what nonsense I'm talking! Really?", + "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", + "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." + ] + ) + } +} From 966b9c19a58c7b8fdfd8a1a04d5909b805341576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 9 May 2022 16:45:37 +0200 Subject: [PATCH 16/46] Tokenize the TTS text spans --- Sources/Navigator/TTS/TTSController.swift | 57 +++++++++++++++---- .../Navigator/Toolkit/Extensions/String.swift | 20 +++++++ .../Services/Content/ContentIterator.swift | 6 ++ 3 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 Sources/Navigator/Toolkit/Extensions/String.swift diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index 97bead63e..ccfdd1228 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -60,9 +60,15 @@ public class TTSController: Loggable, TTSEngineDelegate { private let publication: Publication private let engine: TTSEngine + private let tokenizer: Tokenizer private let queue: DispatchQueue = .global(qos: .userInitiated) - public init(publication: Publication, engine: TTSEngine = AVTTSEngine(), delegate: TTSControllerDelegate? = nil) { + public init( + publication: Publication, + engine: TTSEngine = AVTTSEngine(), + tokenizer: Tokenizer = DefaultTokenizer(unit: .sentence), + delegate: TTSControllerDelegate? = nil + ) { precondition(publication.isContentIterable, "The Publication must be iterable to be used with TTSController") self.defaultRate = engine.defaultRate ?? 0.5 @@ -71,6 +77,7 @@ public class TTSController: Loggable, TTSEngineDelegate { self.delegate = delegate self.publication = publication self.engine = engine + self.tokenizer = tokenizer engine.delegate = self } @@ -233,14 +240,17 @@ public class TTSController: Loggable, TTSEngineDelegate { return Array(ofNotNil: utterance(text: description, locator: content.locator)) case .text(spans: let spans, style: _): - return spans.enumerated().compactMap { offset, span in - utterance( - text: span.text, - locator: span.locator, - language: span.language, - postDelay: (offset == spans.count - 1) ? 0.4 : 0 - ) - } + return spans + .flatMap { tokenize($0) } + .enumerated() + .compactMap { offset, span in + utterance( + text: span.text, + locator: span.locator, + language: span.language, + postDelay: (offset == spans.count - 1) ? 0.4 : 0 + ) + } } } @@ -264,6 +274,33 @@ public class TTSController: Loggable, TTSEngineDelegate { ?? engine.defaultLanguage } + private func tokenize(_ span: Content.TextSpan) -> [Content.TextSpan] { + do { + return try tokenizer + .tokenize(text: span.text) + .map { range in + Content.TextSpan( + locator: span.locator.copy(text: { $0 = self.extractTextContext(in: span.text, for: range) }), + language: span.language, + text: String(span.text[range]) + ) + } + } catch { + log(.error, error) + return [span] + } + } + + private func extractTextContext(in string: String, for range: Range) -> Locator.Text { + let after = String(string[range.upperBound.. String.Index { + precondition(n != 0) + let limit = (n > 0) ? endIndex : startIndex + guard let index = index(i, offsetBy: n, limitedBy: limit) else { + return limit + } + return index + } +} diff --git a/Sources/Shared/Publication/Services/Content/ContentIterator.swift b/Sources/Shared/Publication/Services/Content/ContentIterator.swift index 6be5acd87..584c29c79 100644 --- a/Sources/Shared/Publication/Services/Content/ContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/ContentIterator.swift @@ -43,6 +43,12 @@ public struct Content: Equatable { public let locator: Locator public let language: String? public let text: String + + public init(locator: Locator, language: String?, text: String) { + self.locator = locator + self.language = language + self.text = text + } } } From bfd06f723d0caeb150a5ddf75c8ce3f0ae144480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 9 May 2022 17:56:03 +0200 Subject: [PATCH 17/46] Refactor the text tokanizers into functions and introduce generic `Tokenizer` --- Sources/Navigator/TTS/TTSController.swift | 7 +- .../Toolkit/Tokenizer/NLTokenizer.swift | 47 ----- .../Toolkit/Tokenizer/NSTokenizer.swift | 71 -------- .../Toolkit/Tokenizer/SimpleTokenizer.swift | 45 ----- .../Toolkit/Tokenizer/TextTokenizer.swift | 168 +++++++++++++++++ .../Shared/Toolkit/Tokenizer/Tokenizer.swift | 32 +--- .../Toolkit/Tokenizer/NLTokenizerTests.swift | 62 ------- .../Toolkit/Tokenizer/NSTokenizerTests.swift | 64 ------- .../Tokenizer/SimpleTokenizerTests.swift | 63 ------- .../Tokenizer/TextTokenizerTests.swift | 169 ++++++++++++++++++ 10 files changed, 342 insertions(+), 386 deletions(-) delete mode 100644 Sources/Shared/Toolkit/Tokenizer/NLTokenizer.swift delete mode 100644 Sources/Shared/Toolkit/Tokenizer/NSTokenizer.swift delete mode 100644 Sources/Shared/Toolkit/Tokenizer/SimpleTokenizer.swift create mode 100644 Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift delete mode 100644 Tests/SharedTests/Toolkit/Tokenizer/NLTokenizerTests.swift delete mode 100644 Tests/SharedTests/Toolkit/Tokenizer/NSTokenizerTests.swift delete mode 100644 Tests/SharedTests/Toolkit/Tokenizer/SimpleTokenizerTests.swift create mode 100644 Tests/SharedTests/Toolkit/Tokenizer/TextTokenizerTests.swift diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index ccfdd1228..0a438624f 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -60,13 +60,13 @@ public class TTSController: Loggable, TTSEngineDelegate { private let publication: Publication private let engine: TTSEngine - private let tokenizer: Tokenizer + private let tokenizer: TextTokenizer private let queue: DispatchQueue = .global(qos: .userInitiated) public init( publication: Publication, engine: TTSEngine = AVTTSEngine(), - tokenizer: Tokenizer = DefaultTokenizer(unit: .sentence), + tokenizer: @escaping TextTokenizer = makeDefaultTextTokenizer(unit: .sentence), delegate: TTSControllerDelegate? = nil ) { precondition(publication.isContentIterable, "The Publication must be iterable to be used with TTSController") @@ -276,8 +276,7 @@ public class TTSController: Loggable, TTSEngineDelegate { private func tokenize(_ span: Content.TextSpan) -> [Content.TextSpan] { do { - return try tokenizer - .tokenize(text: span.text) + return try tokenizer(span.text) .map { range in Content.TextSpan( locator: span.locator.copy(text: { $0 = self.extractTextContext(in: span.text, for: range) }), diff --git a/Sources/Shared/Toolkit/Tokenizer/NLTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/NLTokenizer.swift deleted file mode 100644 index f1574884f..000000000 --- a/Sources/Shared/Toolkit/Tokenizer/NLTokenizer.swift +++ /dev/null @@ -1,47 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import Foundation -import NaturalLanguage - -/// A text `Tokenizer` using iOS 12+'s NaturalLanguage framework. -@available(iOS 12.0, *) -public class NLTokenizer: Tokenizer { - private let unit: NLTokenUnit - private let language: NLLanguage? - - public init(unit: TokenUnit, language: String? = nil) { - self.unit = unit.nlUnit - self.language = language.map { NLLanguage($0) } - } - - public func tokenize(text: String) throws -> [Range] { - let tokenizer = NaturalLanguage.NLTokenizer(unit: unit) - tokenizer.string = text - if let language = language { - tokenizer.setLanguage(language) - } - - return tokenizer.tokens(for: text.startIndex.. 0 } - } -} - -private extension TokenUnit { - @available(iOS 12.0, *) - var nlUnit: NLTokenUnit { - switch self { - case .word: - return .word - case .sentence: - return .sentence - case .paragraph: - return .paragraph - } - } -} \ No newline at end of file diff --git a/Sources/Shared/Toolkit/Tokenizer/NSTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/NSTokenizer.swift deleted file mode 100644 index 0551b7150..000000000 --- a/Sources/Shared/Toolkit/Tokenizer/NSTokenizer.swift +++ /dev/null @@ -1,71 +0,0 @@ -// -// Copyright 2021 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import Foundation - -public enum NSTokenizerError: Error { - case rangeConversionFailed(range: NSRange, string: String) -} - -/// A text `Tokenizer` using iOS 11+'s `NSLinguisticTaggerUnit`. -/// -/// Prefer using NLTokenizer on iOS 12+. -@available(iOS 11.0, *) -public class NSTokenizer: Tokenizer, Loggable { - private let unit: NSLinguisticTaggerUnit - private let options: NSLinguisticTagger.Options - - public init( - unit: TokenUnit, - options: NSLinguisticTagger.Options = [.joinNames, .omitPunctuation, .omitWhitespace] - ) { - self.unit = unit.nsUnit - self.options = options - } - - public func tokenize(text: String) throws -> [Range] { - let tagger = NSLinguisticTagger(tagSchemes: [.tokenType], options: 0) - tagger.string = text - - var error: Error? - var tokens: [Range] = [] - tagger.enumerateTags( - in: NSRange(location: 0, length: text.utf16.count), - unit: unit, - scheme: .tokenType, - options: options - ) { _, nsRange, _ in - guard let range = Range(nsRange, in: text) else { - error = NSTokenizerError.rangeConversionFailed(range: nsRange, string: text) - return - } - tokens.append(range) - } - - if let error = error { - throw error - } - - return tokens - .map { $0.trimmingWhitespaces(in: text) } - // Remove empty ranges. - .filter { $0.upperBound.utf16Offset(in: text) - $0.lowerBound.utf16Offset(in: text) > 0 } - } -} - -private extension TokenUnit { - @available(iOS 11.0, *) - var nsUnit: NSLinguisticTaggerUnit { - switch self { - case .word: - return .word - case .sentence: - return .sentence - case .paragraph: - return .paragraph - } - } -} diff --git a/Sources/Shared/Toolkit/Tokenizer/SimpleTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/SimpleTokenizer.swift deleted file mode 100644 index ff1b8addf..000000000 --- a/Sources/Shared/Toolkit/Tokenizer/SimpleTokenizer.swift +++ /dev/null @@ -1,45 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import Foundation - -/// A `Tokenizer` using the basic `NSString.enumerateSubstrings()` API. -/// -/// Prefer using `NLTokenizer` or `NSTokenizer` on more recent versions of iOS. -public class SimpleTokenizer: Tokenizer { - private let options: NSString.EnumerationOptions - - public init(unit: TokenUnit) { - self.options = unit.enumerationOptions.union(.substringNotRequired) - } - - public func tokenize(text: String) throws -> [Range] { - var tokens: [Range] = [] - text.enumerateSubstrings( - in: text.startIndex.. 0 } - } -} - -private extension TokenUnit { - var enumerationOptions: NSString.EnumerationOptions { - switch self { - case .word: - return .byWords - case .sentence: - return .bySentences - case .paragraph: - return .byParagraphs - } - } -} \ No newline at end of file diff --git a/Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift new file mode 100644 index 000000000..3ace0305e --- /dev/null +++ b/Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift @@ -0,0 +1,168 @@ +// +// Copyright 2021 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation +import NaturalLanguage + +/// A tokenizer splitting a String into range tokens (e.g. words, sentences, etc.). +public typealias TextTokenizer = Tokenizer> + +/// A text token unit which can be used with a `TextTokenizer`. +public enum TextUnit { + case word, sentence, paragraph +} + +public enum TextTokenizerError: Error { + case rangeConversionFailed(range: NSRange, string: String) +} + +/// A default cluster `Tokenizer` taking advantage of the best capabilities of each iOS version. +public func makeDefaultTextTokenizer(unit: TextUnit, language: String? = nil) -> TextTokenizer { + if #available(iOS 12.0, *) { + return makeNLTextTokenizer(unit: unit, language: language) + } else if #available(iOS 11.0, *) { + return makeNSTextTokenizer(unit: unit) + } else { + return makeSimpleTextTokenizer(unit: unit) + } +} + + +// MARK: - NL Text Tokenizer + +/// A text `Tokenizer` using iOS 12+'s NaturalLanguage framework. +@available(iOS 12.0, *) +public func makeNLTextTokenizer(unit: TextUnit, language: String? = nil) -> TextTokenizer { + let unit = unit.nlUnit + let language = language.map { NLLanguage($0) } + + func tokenize(_ text: String) throws -> [Range] { + let tokenizer = NLTokenizer(unit: unit) + tokenizer.string = text + if let language = language { + tokenizer.setLanguage(language) + } + + return tokenizer.tokens(for: text.startIndex.. 0 } + } + + return tokenize +} + +private extension TextUnit { + @available(iOS 12.0, *) + var nlUnit: NLTokenUnit { + switch self { + case .word: + return .word + case .sentence: + return .sentence + case .paragraph: + return .paragraph + } + } +} + + +// MARK: - NS Text Tokenizer + +/// A text `Tokenizer` using iOS 11+'s `NSLinguisticTaggerUnit`. +/// +/// Prefer using NLTokenizer on iOS 12+. +@available(iOS 11.0, *) +public func makeNSTextTokenizer( + unit: TextUnit, + options: NSLinguisticTagger.Options = [.joinNames, .omitPunctuation, .omitWhitespace] +) -> TextTokenizer { + let unit = unit.nsUnit + + func tokenize(_ text: String) throws -> [Range] { + let tagger = NSLinguisticTagger(tagSchemes: [.tokenType], options: 0) + tagger.string = text + + var error: Error? + var tokens: [Range] = [] + tagger.enumerateTags( + in: NSRange(location: 0, length: text.utf16.count), + unit: unit, + scheme: .tokenType, + options: options + ) { _, nsRange, _ in + guard let range = Range(nsRange, in: text) else { + error = TextTokenizerError.rangeConversionFailed(range: nsRange, string: text) + return + } + tokens.append(range) + } + + if let error = error { + throw error + } + + return tokens + .map { $0.trimmingWhitespaces(in: text) } + // Remove empty ranges. + .filter { $0.upperBound.utf16Offset(in: text) - $0.lowerBound.utf16Offset(in: text) > 0 } + } + + return tokenize +} + +private extension TextUnit { + @available(iOS 11.0, *) + var nsUnit: NSLinguisticTaggerUnit { + switch self { + case .word: + return .word + case .sentence: + return .sentence + case .paragraph: + return .paragraph + } + } +} + + +// MARK: - Simple Text Tokenizer + +/// A `Tokenizer` using the basic `NSString.enumerateSubstrings()` API. +/// +/// Prefer using `NLTokenizer` or `NSTokenizer` on more recent versions of iOS. +public func makeSimpleTextTokenizer(unit: TextUnit) -> TextTokenizer { + let options = unit.enumerationOptions.union(.substringNotRequired) + + func tokenize(_ text: String) throws -> [Range] { + var tokens: [Range] = [] + text.enumerateSubstrings( + in: text.startIndex.. 0 } + } + + return tokenize +} + +private extension TextUnit { + var enumerationOptions: NSString.EnumerationOptions { + switch self { + case .word: + return .byWords + case .sentence: + return .bySentences + case .paragraph: + return .byParagraphs + } + } +} diff --git a/Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift index 8bb2be91b..89b2fdf04 100644 --- a/Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift +++ b/Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift @@ -6,33 +6,5 @@ import Foundation -/// A tokenizer splits a String into tokens (e.g. words, sentences, etc.). -public protocol Tokenizer { - - /// Splits the given `text` into tokens and return their range in the string. - func tokenize(text: String) throws -> [Range] -} - -/// A text token unit which can be used with a `Tokenizer`. -public enum TokenUnit { - case word, sentence, paragraph -} - -/// A default cluster `Tokenizer` taking advantage of the best capabilities of each iOS version. -public class DefaultTokenizer: Tokenizer { - private let tokenizer: Tokenizer - - public init(unit: TokenUnit, language: String? = nil) { - if #available(iOS 12.0, *) { - tokenizer = NLTokenizer(unit: unit, language: language) - } else if #available(iOS 11.0, *) { - tokenizer = NSTokenizer(unit: unit) - } else { - tokenizer = SimpleTokenizer(unit: unit) - } - } - - public func tokenize(text: String) throws -> [Range] { - try tokenizer.tokenize(text: text) - } -} \ No newline at end of file +/// A tokenizer splits a content into a list of tokens. +public typealias Tokenizer = (_ data: Data) throws -> [Token] diff --git a/Tests/SharedTests/Toolkit/Tokenizer/NLTokenizerTests.swift b/Tests/SharedTests/Toolkit/Tokenizer/NLTokenizerTests.swift deleted file mode 100644 index 995b35d0e..000000000 --- a/Tests/SharedTests/Toolkit/Tokenizer/NLTokenizerTests.swift +++ /dev/null @@ -1,62 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import XCTest -@testable import R2Shared - -@available(iOS 12.0, *) -class NLTokenizerTests: XCTestCase { - - func testTokenizeEmptyText() { - let tokenizer = NLTokenizer(unit: .word) - XCTAssertEqual(try tokenizer.tokenize(text: ""), []) - } - - func testTokenizeByWords() { - let tokenizer = NLTokenizer(unit: .word) - let text = "He said: \n\"What?\"" - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - ["He", "said", "What"] - ) - } - - func testTokenizeBySentences() { - let tokenizer = NLTokenizer(unit: .sentence) - let text = - """ - Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble - In the end, she went ahead. - """ - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - [ - "Mr. Bougee said, looking above: \"and what is the use of a book?\".", - "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", - "In the end, she went ahead." - ] - ) - } - - func testTokenizeByParagraphs() { - let tokenizer = NLTokenizer(unit: .paragraph) - let text = - """ - Oh dear, what nonsense I'm talking! Really? - - Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. - Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. - """ - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - [ - "Oh dear, what nonsense I'm talking! Really?", - "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", - "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." - ] - ) - } -} diff --git a/Tests/SharedTests/Toolkit/Tokenizer/NSTokenizerTests.swift b/Tests/SharedTests/Toolkit/Tokenizer/NSTokenizerTests.swift deleted file mode 100644 index 7a86cb6b4..000000000 --- a/Tests/SharedTests/Toolkit/Tokenizer/NSTokenizerTests.swift +++ /dev/null @@ -1,64 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import XCTest -@testable import R2Shared - -import NaturalLanguage - -@available(iOS 12.0, *) -class NSTokenizerTests: XCTestCase { - - func testTokenizeEmptyText() { - let tokenizer = NSTokenizer(unit: .word) - XCTAssertEqual(try tokenizer.tokenize(text: ""), []) - } - - func testTokenizeByWords() { - let tokenizer = NSTokenizer(unit: .word) - let text = "He said: \n\"What?\"" - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - ["He", "said", "What"] - ) - } - - func testTokenizeBySentences() { - let tokenizer = NSTokenizer(unit: .sentence) - let text = - """ - Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble - In the end, she went ahead. - """ - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - [ - "Mr. Bougee said, looking above: \"and what is the use of a book?\".", - "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", - "In the end, she went ahead." - ] - ) - } - - func testTokenizeByParagraphs() { - let tokenizer = NSTokenizer(unit: .paragraph) - let text = - """ - Oh dear, what nonsense I'm talking! Really? - - Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. - Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. - """ - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - [ - "Oh dear, what nonsense I'm talking! Really?", - "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", - "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." - ] - ) - } -} diff --git a/Tests/SharedTests/Toolkit/Tokenizer/SimpleTokenizerTests.swift b/Tests/SharedTests/Toolkit/Tokenizer/SimpleTokenizerTests.swift deleted file mode 100644 index 819d73895..000000000 --- a/Tests/SharedTests/Toolkit/Tokenizer/SimpleTokenizerTests.swift +++ /dev/null @@ -1,63 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import XCTest -@testable import R2Shared - -@available(iOS 12.0, *) -class SimpleTokenizerTests: XCTestCase { - - func testTokenizeEmptyText() { - let tokenizer = SimpleTokenizer(unit: .word) - XCTAssertEqual(try tokenizer.tokenize(text: ""), []) - } - - func testTokenizeByWords() { - let tokenizer = SimpleTokenizer(unit: .word) - let text = "He said: \n\"What?\"" - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - ["He", "said", "What"] - ) - } - - func testTokenizeBySentences() { - let tokenizer = SimpleTokenizer(unit: .sentence) - let text = - """ - Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble - In the end, she went ahead. - """ - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - [ - "Mr.", - "Bougee said, looking above: \"and what is the use of a book?\".", - "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", - "In the end, she went ahead." - ] - ) - } - - func testTokenizeByParagraphs() { - let tokenizer = SimpleTokenizer(unit: .paragraph) - let text = - """ - Oh dear, what nonsense I'm talking! Really? - - Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. - Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. - """ - XCTAssertEqual( - try tokenizer.tokenize(text: text).map { String(text[$0]) }, - [ - "Oh dear, what nonsense I'm talking! Really?", - "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", - "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." - ] - ) - } -} diff --git a/Tests/SharedTests/Toolkit/Tokenizer/TextTokenizerTests.swift b/Tests/SharedTests/Toolkit/Tokenizer/TextTokenizerTests.swift new file mode 100644 index 000000000..2066600e9 --- /dev/null +++ b/Tests/SharedTests/Toolkit/Tokenizer/TextTokenizerTests.swift @@ -0,0 +1,169 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import XCTest +@testable import R2Shared + +@available(iOS 12.0, *) +class TextTokenizerTests: XCTestCase { + + // MARK: - NL + + func testNLTokenizeEmptyText() { + let tokenizer = makeNLTextTokenizer(unit: .word) + XCTAssertEqual(try tokenizer(""), []) + } + + func testNLTokenizeByWords() { + let tokenizer = makeNLTextTokenizer(unit: .word) + let text = "He said: \n\"What?\"" + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + ["He", "said", "What"] + ) + } + + func testNLTokenizeBySentences() { + let tokenizer = makeNLTextTokenizer(unit: .sentence) + let text = + """ + Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble + In the end, she went ahead. + """ + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + [ + "Mr. Bougee said, looking above: \"and what is the use of a book?\".", + "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", + "In the end, she went ahead." + ] + ) + } + + func testNLTokenizeByParagraphs() { + let tokenizer = makeNLTextTokenizer(unit: .paragraph) + let text = + """ + Oh dear, what nonsense I'm talking! Really? + + Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. + Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. + """ + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + [ + "Oh dear, what nonsense I'm talking! Really?", + "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", + "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." + ] + ) + } + + // MARK: - NS + + func testNSTokenizeEmptyText() { + let tokenizer = makeNSTextTokenizer(unit: .word) + XCTAssertEqual(try tokenizer(""), []) + } + + func testNSTokenizeByWords() { + let tokenizer = makeNSTextTokenizer(unit: .word) + let text = "He said: \n\"What?\"" + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + ["He", "said", "What"] + ) + } + + func testNSTokenizeBySentences() { + let tokenizer = makeNSTextTokenizer(unit: .sentence) + let text = + """ + Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble + In the end, she went ahead. + """ + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + [ + "Mr. Bougee said, looking above: \"and what is the use of a book?\".", + "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", + "In the end, she went ahead." + ] + ) + } + + func testNSTokenizeByParagraphs() { + let tokenizer = makeNSTextTokenizer(unit: .paragraph) + let text = + """ + Oh dear, what nonsense I'm talking! Really? + + Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. + Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. + """ + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + [ + "Oh dear, what nonsense I'm talking! Really?", + "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", + "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." + ] + ) + } + + // MARK: - Simple + + func testSimpleTokenizeEmptyText() { + let tokenizer = makeSimpleTextTokenizer(unit: .word) + XCTAssertEqual(try tokenizer(""), []) + } + + func testSimpleTokenizeByWords() { + let tokenizer = makeSimpleTextTokenizer(unit: .word) + let text = "He said: \n\"What?\"" + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + ["He", "said", "What"] + ) + } + + func testSimpleTokenizeBySentences() { + let tokenizer = makeSimpleTextTokenizer(unit: .sentence) + let text = + """ + Mr. Bougee said, looking above: "and what is the use of a book?". So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble + In the end, she went ahead. + """ + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + [ + "Mr.", + "Bougee said, looking above: \"and what is the use of a book?\".", + "So she was considering (as well as she could), whether making a daisy-chain would be worth the trouble", + "In the end, she went ahead." + ] + ) + } + + func testSimpleTokenizeByParagraphs() { + let tokenizer = makeSimpleTextTokenizer(unit: .paragraph) + let text = + """ + Oh dear, what nonsense I'm talking! Really? + + Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door. + Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again. + """ + XCTAssertEqual( + try tokenizer(text).map { String(text[$0]) }, + [ + "Oh dear, what nonsense I'm talking! Really?", + "Just then her head struck against the roof of the hall: in fact she was now more than nine feet high, and she at once took up the little golden key and hurried off to the garden door.", + "Poor Alice! It was as much as she could do, lying down on one side, to look through into the garden with one eye; but to get through was more hopeless than ever: she sat down and began to cry again." + ] + ) + } +} From 9ae1b7d49507436097922c9169cad37baa01665a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 9 May 2022 19:16:37 +0200 Subject: [PATCH 18/46] Add the `ContentTokenizer` --- Sources/Navigator/TTS/TTSController.swift | 40 +++++-------- .../Navigator/Toolkit/Extensions/String.swift | 20 ------- .../Shared/Toolkit/Extensions/String.swift | 10 ++++ .../Toolkit/Tokenizer/ContentTokenizer.swift | 56 +++++++++++++++++++ 4 files changed, 81 insertions(+), 45 deletions(-) delete mode 100644 Sources/Navigator/Toolkit/Extensions/String.swift create mode 100644 Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index 0a438624f..8fd9fe583 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -32,6 +32,7 @@ public struct TTSUtterance: Equatable { } public class TTSController: Loggable, TTSEngineDelegate { + public typealias TokenizerFactory = (_ language: String?) -> ContentTokenizer public struct Configuration { public var defaultLanguage: String? @@ -60,13 +61,13 @@ public class TTSController: Loggable, TTSEngineDelegate { private let publication: Publication private let engine: TTSEngine - private let tokenizer: TextTokenizer + private let makeTokenizer: TokenizerFactory private let queue: DispatchQueue = .global(qos: .userInitiated) public init( publication: Publication, engine: TTSEngine = AVTTSEngine(), - tokenizer: @escaping TextTokenizer = makeDefaultTextTokenizer(unit: .sentence), + tokenizerFactory: TokenizerFactory? = nil, delegate: TTSControllerDelegate? = nil ) { precondition(publication.isContentIterable, "The Publication must be iterable to be used with TTSController") @@ -77,7 +78,12 @@ public class TTSController: Loggable, TTSEngineDelegate { self.delegate = delegate self.publication = publication self.engine = engine - self.tokenizer = tokenizer + self.makeTokenizer = { language in + makeTextContentTokenizer( + unit: .sentence, + language: language + ) + } engine.delegate = self } @@ -220,7 +226,9 @@ public class TTSController: Loggable, TTSEngineDelegate { return false } - utterances = utterances(from: content) + utterances = tokenize(content, with: makeTokenizer(defaultLanguage)) + .flatMap { utterances(from: $0) } + guard !utterances.isEmpty else { return try loadNextUtterances(direction: direction) } @@ -241,7 +249,6 @@ public class TTSController: Loggable, TTSEngineDelegate { case .text(spans: let spans, style: _): return spans - .flatMap { tokenize($0) } .enumerated() .compactMap { offset, span in utterance( @@ -274,32 +281,15 @@ public class TTSController: Loggable, TTSEngineDelegate { ?? engine.defaultLanguage } - private func tokenize(_ span: Content.TextSpan) -> [Content.TextSpan] { + private func tokenize(_ content: Content, with tokenizer: ContentTokenizer) -> [Content] { do { - return try tokenizer(span.text) - .map { range in - Content.TextSpan( - locator: span.locator.copy(text: { $0 = self.extractTextContext(in: span.text, for: range) }), - language: span.language, - text: String(span.text[range]) - ) - } + return try tokenizer(content) } catch { log(.error, error) - return [span] + return [content] } } - private func extractTextContext(in string: String, for range: Range) -> Locator.Text { - let after = String(string[range.upperBound.. String.Index { - precondition(n != 0) - let limit = (n > 0) ? endIndex : startIndex - guard let index = index(i, offsetBy: n, limitedBy: limit) else { - return limit - } - return index - } -} diff --git a/Sources/Shared/Toolkit/Extensions/String.swift b/Sources/Shared/Toolkit/Extensions/String.swift index 6d7d93d90..9b9cb5024 100644 --- a/Sources/Shared/Toolkit/Extensions/String.swift +++ b/Sources/Shared/Toolkit/Extensions/String.swift @@ -59,4 +59,14 @@ extension String { func coalescingWhitespaces() -> String { replacingOccurrences(of: "[\\s\n]+", with: " ", options: .regularExpression, range: nil) } + + /// Same as `index(_,offsetBy:)` but without crashing when reaching the end of the string. + func clampedIndex(_ i: String.Index, offsetBy n: String.IndexDistance) -> String.Index { + precondition(n != 0) + let limit = (n > 0) ? endIndex : startIndex + guard let index = index(i, offsetBy: n, limitedBy: limit) else { + return limit + } + return index + } } diff --git a/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift new file mode 100644 index 000000000..ba678e41a --- /dev/null +++ b/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift @@ -0,0 +1,56 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// A tokenizer splitting a `Content` into smaller pieces. +public typealias ContentTokenizer = Tokenizer + +/// A `ContentTokenizer` using the default `TextTokenizer` to split the text of the `Content` by `unit`. +public func makeTextContentTokenizer(unit: TextUnit, language: String?) -> ContentTokenizer { + makeTextContentTokenizer(with: makeDefaultTextTokenizer(unit: unit, language: language)) +} + +/// A `ContentTokenizer` using a `TextTokenizer` to split the text of the `Content`. +public func makeTextContentTokenizer(with tokenizer: @escaping TextTokenizer) -> ContentTokenizer { + func tokenize(_ span: Content.TextSpan) throws -> [Content.TextSpan] { + try tokenizer(span.text) + .map { range in + Content.TextSpan( + locator: span.locator.copy(text: { $0 = extractTextContext(in: span.text, for: range) }), + language: span.language, + text: String(span.text[range]) + ) + } + } + + func tokenize(_ content: Content) throws -> [Content] { + switch content.data { + case .audio, .image: + return [content] + case .text(spans: let spans, style: let style): + return [Content( + locator: content.locator, + data: .text( + spans: try spans.flatMap { try tokenize($0) }, + style: style + ) + )] + } + } + + return tokenize +} + +private func extractTextContext(in string: String, for range: Range) -> Locator.Text { + let after = String(string[range.upperBound.. Date: Tue, 10 May 2022 12:29:24 +0200 Subject: [PATCH 19/46] Deprecate `Locator(link: Link)` in favor of `Publication.locate(Link)` (#47) --- .gitignore | 2 +- CHANGELOG.md | 7 ++ Makefile | 6 ++ .../Navigator/Audiobook/AudioNavigator.swift | 7 +- .../CBZ/CBZNavigatorViewController.swift | 2 +- .../EPUB/EPUBNavigatorViewController.swift | 7 +- Sources/Shared/Publication/Locator.swift | 3 +- Sources/Shared/Publication/Manifest.swift | 31 ++++++- Sources/Shared/Publication/Publication.swift | 36 ++------ .../Locator/DefaultLocatorService.swift | 67 +++++++++++---- .../Services/Locator/LocatorService.swift | 24 ++++-- .../Positions/InMemoryPositionsService.swift | 23 ++++++ .../PerResourcePositionsService.swift | 12 +-- .../Services/PublicationServicesBuilder.swift | 2 +- .../Services/Search/StringSearchService.swift | 16 ++-- Sources/Shared/Toolkit/DocumentTypes.swift | 22 +++-- Sources/Shared/Toolkit/Weak.swift | 24 +++++- .../Audio/Services/AudioLocatorService.swift | 55 +++++-------- .../Readium.xcodeproj/project.pbxproj | 4 + TestApp/.gitignore | 2 + .../Library/LibraryViewController.swift | 19 ++--- .../OPDSCatalogSelectorViewController.swift | 21 +++-- .../Common/Outline/OutlineTableView.swift | 17 ++-- .../Publication/LocatorTests.swift | 41 ---------- .../Locator/DefaultLocatorServiceTests.swift | 82 ++++++++++++++++++- .../Services/AudioLocatorServiceTests.swift | 4 +- 26 files changed, 343 insertions(+), 193 deletions(-) create mode 100644 Sources/Shared/Publication/Services/Positions/InMemoryPositionsService.swift diff --git a/.gitignore b/.gitignore index ba719ac08..0c5fcb4e6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ Package.resolved # Carthage -Carthage/ +./Carthage/ Cartfile.resolved # Xcode diff --git a/CHANGELOG.md b/CHANGELOG.md index 85dd81e59..b2519510b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project will be documented in this file. Take a look ## [Unreleased] +### Deprecated + +#### Shared + +* `Locator(link: Link)` is deprecated as it may create an incorrect `Locator` if the link `type` is missing. + * Use `publication.locate(Link)` instead. + ### Fixed #### Navigator diff --git a/Makefile b/Makefile index 18ef3ac3a..cbbe29f95 100644 --- a/Makefile +++ b/Makefile @@ -16,3 +16,9 @@ scripts: yarn --cwd "$(SCRIPTS_PATH)" run format yarn --cwd "$(SCRIPTS_PATH)" run lint yarn --cwd "$(SCRIPTS_PATH)" run bundle + +.PHONY: test +test: + # To limit to a particular test suite: -only-testing:R2SharedTests + xcodebuild test -scheme "Readium-Package" -destination "platform=iOS Simulator,name=iPhone 12" | xcbeautify -q + diff --git a/Sources/Navigator/Audiobook/AudioNavigator.swift b/Sources/Navigator/Audiobook/AudioNavigator.swift index b4a741be9..af256f547 100644 --- a/Sources/Navigator/Audiobook/AudioNavigator.swift +++ b/Sources/Navigator/Audiobook/AudioNavigator.swift @@ -27,7 +27,7 @@ open class _AudioNavigator: _MediaNavigator, _AudioSessionUser, Loggable { public init(publication: Publication, initialLocation: Locator? = nil) { self.publication = publication self.initialLocation = initialLocation - ?? publication.readingOrder.first.map { Locator(link: $0) } + ?? publication.readingOrder.first.flatMap { publication.locate($0) } let durations = publication.readingOrder.map { $0.duration ?? 0 } self.durations = durations @@ -255,7 +255,10 @@ open class _AudioNavigator: _MediaNavigator, _AudioSessionUser, Loggable { @discardableResult public func go(to link: Link, animated: Bool = false, completion: @escaping () -> Void = {}) -> Bool { - return go(to: Locator(link: link), animated: animated, completion: completion) + guard let locator = publication.locate(link) else { + return false + } + return go(to: locator, animated: animated, completion: completion) } @discardableResult diff --git a/Sources/Navigator/CBZ/CBZNavigatorViewController.swift b/Sources/Navigator/CBZ/CBZNavigatorViewController.swift index 979a0b476..76335cbb1 100644 --- a/Sources/Navigator/CBZ/CBZNavigatorViewController.swift +++ b/Sources/Navigator/CBZ/CBZNavigatorViewController.swift @@ -232,7 +232,7 @@ extension CBZNavigatorViewController { public convenience init(for publication: Publication, initialIndex: Int = 0) { var location: Locator? = nil if publication.readingOrder.indices.contains(initialIndex) { - location = Locator(link: publication.readingOrder[initialIndex]) + location = publication.locate(publication.readingOrder[initialIndex]) } self.init(publication: publication, initialLocation: location) } diff --git a/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift b/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift index 75eaea02f..e296b8ed9 100644 --- a/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift +++ b/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift @@ -520,7 +520,7 @@ open class EPUBNavigatorViewController: UIViewController, VisualNavigator, Selec locations: { $0.progression = progression } ) } else { - return Locator(link: link).copy( + return publication.locate(link)?.copy( locations: { $0.progression = progression } ) } @@ -564,7 +564,10 @@ open class EPUBNavigatorViewController: UIViewController, VisualNavigator, Selec } public func go(to link: Link, animated: Bool, completion: @escaping () -> Void) -> Bool { - return go(to: Locator(link: link), animated: animated, completion: completion) + guard let locator = publication.locate(link) else { + return false + } + return go(to: locator, animated: animated, completion: completion) } public func goForward(animated: Bool, completion: @escaping () -> Void) -> Bool { diff --git a/Sources/Shared/Publication/Locator.swift b/Sources/Shared/Publication/Locator.swift index b787b0216..4d57543cb 100644 --- a/Sources/Shared/Publication/Locator.swift +++ b/Sources/Shared/Publication/Locator.swift @@ -70,7 +70,8 @@ public struct Locator: Hashable, CustomStringConvertible, Loggable { try self.init(json: json, warnings: warnings) } - + + @available(*, deprecated, message: "This may create an incorrect `Locator` if the link `type` is missing. Use `publication.locate(Link)` instead.") public init(link: Link) { let components = link.href.split(separator: "#", maxSplits: 1).map(String.init) let fragments = (components.count > 1) ? [String(components[1])] : [] diff --git a/Sources/Shared/Publication/Manifest.swift b/Sources/Shared/Publication/Manifest.swift index afd0c8b15..e3287ba96 100644 --- a/Sources/Shared/Publication/Manifest.swift +++ b/Sources/Shared/Publication/Manifest.swift @@ -126,7 +126,36 @@ public struct Manifest: JSONEquatable, Hashable { return metadata.conformsTo.contains(profile) } - + + /// Finds the first Link having the given `href` in the manifest's links. + public func link(withHREF href: String) -> Link? { + func deepFind(in linkLists: [Link]...) -> Link? { + for links in linkLists { + for link in links { + if link.href == href { + return link + } else if let child = deepFind(in: link.alternates, link.children) { + return child + } + } + } + + return nil + } + + var link = deepFind(in: readingOrder, resources, links) + if + link == nil, + let shortHREF = href.components(separatedBy: .init(charactersIn: "#?")).first, + shortHREF != href + { + // Tries again, but without the anchor and query parameters. + link = self.link(withHREF: shortHREF) + } + + return link + } + /// Finds the first link with the given relation in the manifest's links. public func link(withRel rel: LinkRelation) -> Link? { return readingOrder.first(withRel: rel) diff --git a/Sources/Shared/Publication/Publication.swift b/Sources/Shared/Publication/Publication.swift index dc55f15b6..74f7e60db 100644 --- a/Sources/Shared/Publication/Publication.swift +++ b/Sources/Shared/Publication/Publication.swift @@ -84,7 +84,7 @@ public class Publication: Loggable { /// Returns whether this publication conforms to the given Readium Web Publication Profile. public func conforms(to profile: Profile) -> Bool { - return manifest.conforms(to: profile) + manifest.conforms(to: profile) } /// The URL where this publication is served, computed from the `Link` with `self` relation. @@ -97,41 +97,17 @@ public class Publication: Loggable { /// Finds the first Link having the given `href` in the publication's links. public func link(withHREF href: String) -> Link? { - func deepFind(in linkLists: [Link]...) -> Link? { - for links in linkLists { - for link in links { - if link.href == href { - return link - } else if let child = deepFind(in: link.alternates, link.children) { - return child - } - } - } - - return nil - } - - var link = deepFind(in: readingOrder, resources, links) - if - link == nil, - let shortHREF = href.components(separatedBy: .init(charactersIn: "#?")).first, - shortHREF != href - { - // Tries again, but without the anchor and query parameters. - link = self.link(withHREF: shortHREF) - } - - return link + manifest.link(withHREF: href) } /// Finds the first link with the given relation in the publication's links. public func link(withRel rel: LinkRelation) -> Link? { - return manifest.link(withRel: rel) + manifest.link(withRel: rel) } /// Finds all the links with the given relation in the publication's links. public func links(withRel rel: LinkRelation) -> [Link] { - return manifest.links(withRel: rel) + manifest.links(withRel: rel) } /// Returns the resource targeted by the given `link`. @@ -161,12 +137,12 @@ public class Publication: Loggable { /// /// e.g. `findService(PositionsService.self)` public func findService(_ serviceType: T.Type) -> T? { - return services.first { $0 is T } as? T + services.first { $0 is T } as? T } /// Finds all the services implementing the given service type. public func findServices(_ serviceType: T.Type) -> [T] { - return services.filter { $0 is T } as! [T] + services.filter { $0 is T } as! [T] } /// Sets the URL where this `Publication`'s RWPM manifest is served. diff --git a/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift b/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift index 4f98b21bb..026812efa 100644 --- a/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift +++ b/Sources/Shared/Publication/Services/Locator/DefaultLocatorService.swift @@ -8,35 +8,70 @@ import Foundation /// A default implementation of the `LocatorService` using the `PositionsService` to locate its inputs. open class DefaultLocatorService: LocatorService, Loggable { - - let readingOrder: [Link] - let positionsByReadingOrder: () -> [[Locator]] - - public init(readingOrder: [Link], positionsByReadingOrder: @escaping () -> [[Locator]]) { - self.readingOrder = readingOrder - self.positionsByReadingOrder = positionsByReadingOrder - } - public convenience init(readingOrder: [Link], publication: Weak) { - self.init(readingOrder: readingOrder, positionsByReadingOrder: { publication()?.positionsByReadingOrder ?? [] }) + public let publication: Weak + + public init(publication: Weak) { + self.publication = publication } + /// Locates the target of the given `locator`. + /// + /// If `locator.href` can be found in the links, `locator` will be returned directly. + /// Otherwise, will attempt to find the closest match using `totalProgression`, `position`, + /// `fragments`, etc. open func locate(_ locator: Locator) -> Locator? { - guard readingOrder.firstIndex(withHREF: locator.href) != nil else { + guard let publication = publication() else { return nil } - - return locator + + if publication.link(withHREF: locator.href) != nil { + return locator + } + + if let totalProgression = locator.locations.totalProgression, let target = locate(progression: totalProgression) { + return target.copy( + title: locator.title, + text: { $0 = locator.text } + ) + } + + return nil } - + + open func locate(_ link: Link) -> Locator? { + let components = link.href.split(separator: "#", maxSplits: 1).map(String.init) + let href = components.first ?? link.href + let fragment = components.getOrNil(1) + + guard + let resourceLink = publication()?.link(withHREF: href), + let type = resourceLink.type + else { + return nil + } + + return Locator( + href: href, + type: type, + title: resourceLink.title ?? link.title, + locations: Locator.Locations( + fragments: Array(ofNotNil: fragment), + progression: (fragment == nil) ? 0.0 : nil + ) + ) + } + open func locate(progression totalProgression: Double) -> Locator? { guard 0.0...1.0 ~= totalProgression else { log(.error, "Progression must be between 0.0 and 1.0, received \(totalProgression)") return nil } - let positions = positionsByReadingOrder() - guard let (readingOrderIndex, position) = findClosest(to: totalProgression, in: positions) else { + guard + let positions = publication()?.positionsByReadingOrder, + let (readingOrderIndex, position) = findClosest(to: totalProgression, in: positions) + else { return nil } diff --git a/Sources/Shared/Publication/Services/Locator/LocatorService.swift b/Sources/Shared/Publication/Services/Locator/LocatorService.swift index 4d076a1a5..1cd7bfaaa 100644 --- a/Sources/Shared/Publication/Services/Locator/LocatorService.swift +++ b/Sources/Shared/Publication/Services/Locator/LocatorService.swift @@ -16,13 +16,21 @@ public typealias LocatorServiceFactory = (PublicationServiceContext) -> LocatorS /// - Converting a `Locator` which was created from an alternate manifest with a different reading /// order. For example, when downloading a streamed manifest or offloading a package. public protocol LocatorService: PublicationService { - + /// Locates the target of the given `locator`. func locate(_ locator: Locator) -> Locator? - + + /// Locates the target of the given `link`. + func locate(_ link: Link) -> Locator? + /// Locates the target at the given `progression` relative to the whole publication. func locate(progression: Double) -> Locator? - +} + +public extension LocatorService { + func locate(_ locator: Locator) -> Locator? { nil } + func locate(_ link: Link) -> Locator? { nil } + func locate(progression: Double) -> Locator? { nil } } @@ -31,19 +39,19 @@ public protocol LocatorService: PublicationService { public extension Publication { /// Locates the target of the given `locator`. - /// - /// If `locator.href` can be found in the reading order, `locator` will be returned directly. - /// Otherwise, will attempt to find the closest match using `totalProgression`, `position`, - /// `fragments`, etc. func locate(_ locator: Locator) -> Locator? { findService(LocatorService.self)?.locate(locator) } - + /// Locates the target at the given `progression` relative to the whole publication. func locate(progression: Double) -> Locator? { findService(LocatorService.self)?.locate(progression: progression) } + /// Locates the target of the given `link`. + func locate(_ link: Link) -> Locator? { + findService(LocatorService.self)?.locate(link) + } } diff --git a/Sources/Shared/Publication/Services/Positions/InMemoryPositionsService.swift b/Sources/Shared/Publication/Services/Positions/InMemoryPositionsService.swift new file mode 100644 index 000000000..e99b0e4eb --- /dev/null +++ b/Sources/Shared/Publication/Services/Positions/InMemoryPositionsService.swift @@ -0,0 +1,23 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// A [PositionsService] holding the pre-computed position locators in memory. +public class InMemoryPositionsService : PositionsService { + + public private(set) var positionsByReadingOrder: [[Locator]] + + public init(positionsByReadingOrder: [[Locator]]) { + self.positionsByReadingOrder = positionsByReadingOrder + } + + public static func makeFactory(positionsByReadingOrder: [[Locator]]) -> (PublicationServiceContext) -> InMemoryPositionsService { + { _ in + InMemoryPositionsService(positionsByReadingOrder: positionsByReadingOrder) + } + } +} diff --git a/Sources/Shared/Publication/Services/Positions/PerResourcePositionsService.swift b/Sources/Shared/Publication/Services/Positions/PerResourcePositionsService.swift index 49a40d272..055c39b9d 100644 --- a/Sources/Shared/Publication/Services/Positions/PerResourcePositionsService.swift +++ b/Sources/Shared/Publication/Services/Positions/PerResourcePositionsService.swift @@ -1,12 +1,7 @@ // -// PerResourcePositionsService.swift -// r2-shared-swift -// -// Created by Mickaël Menu on 01/06/2020. -// -// Copyright 2020 Readium Foundation. All rights reserved. -// Use of this source code is governed by a BSD-style license which is detailed -// in the LICENSE file present in the project repository where this source code is maintained. +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. // import Foundation @@ -46,6 +41,5 @@ public final class PerResourcePositionsService: PositionsService { PerResourcePositionsService(readingOrder: context.manifest.readingOrder, fallbackMediaType: fallbackMediaType) } } - } diff --git a/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift b/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift index 8e3e01353..2004e4533 100644 --- a/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift +++ b/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift @@ -21,7 +21,7 @@ public struct PublicationServicesBuilder { public init( contentProtection: ContentProtectionServiceFactory? = nil, cover: CoverServiceFactory? = nil, - locator: LocatorServiceFactory? = { DefaultLocatorService(readingOrder: $0.manifest.readingOrder, publication: $0.publication) }, + locator: LocatorServiceFactory? = { DefaultLocatorService(publication: $0.publication) }, positions: PositionsServiceFactory? = nil, search: SearchServiceFactory? = nil, setup: (inout PublicationServicesBuilder) -> Void = { _ in } diff --git a/Sources/Shared/Publication/Services/Search/StringSearchService.swift b/Sources/Shared/Publication/Services/Search/StringSearchService.swift index aa36b8a7d..cf7674a21 100644 --- a/Sources/Shared/Publication/Services/Search/StringSearchService.swift +++ b/Sources/Shared/Publication/Services/Search/StringSearchService.swift @@ -154,16 +154,22 @@ public class _StringSearchService: _SearchService { } private func findLocators(in link: Link, resourceIndex: Int, text: String, cancellable: CancellableObject) -> [Locator] { - guard !text.isEmpty else { + guard + !text.isEmpty, + var resourceLocator = publication.locate(link) + else { return [] } - let currentLocale = options.language.map { Locale(identifier: $0) } ?? locale - let title = publication.tableOfContents.titleMatchingHREF(link.href) ?? link.title - let resourceLocator = Locator(link: link).copy(title: title) + let title = publication.tableOfContents.titleMatchingHREF(link.href) + resourceLocator = resourceLocator.copy( + title: Optional(title ?? link.title) + ) var locators: [Locator] = [] + let currentLocale = options.language.map { Locale(identifier: $0) } ?? locale + for range in searchAlgorithm.findRanges(of: query, options: options, in: text, locale: currentLocale, cancellable: cancellable) { guard !cancellable.isCancelled else { return locators @@ -303,4 +309,4 @@ fileprivate extension Link { } return children.titleMatchingHREF(targetHREF) } -} \ No newline at end of file +} diff --git a/Sources/Shared/Toolkit/DocumentTypes.swift b/Sources/Shared/Toolkit/DocumentTypes.swift index d44143411..8f366162b 100644 --- a/Sources/Shared/Toolkit/DocumentTypes.swift +++ b/Sources/Shared/Toolkit/DocumentTypes.swift @@ -1,17 +1,17 @@ // -// DocumentTypes.swift -// r2-shared-swift -// -// Created by Mickaël Menu on 26.06.19. -// -// Copyright 2019 Readium Foundation. All rights reserved. -// Use of this source code is governed by a BSD-style license which is detailed -// in the LICENSE file present in the project repository where this source code is maintained. +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. // import CoreServices import Foundation +#if canImport(UniformTypeIdentifiers) +import UniformTypeIdentifiers +#endif + + /// Provides a convenient access layer to the Document Types declared in the `Info.plist`, /// under `CFBundleDocumentTypes`. public struct DocumentTypes { @@ -25,6 +25,12 @@ public struct DocumentTypes { /// Supported UTIs. public let supportedUTIs: [String] + /// Supported UTTypes. + @available(iOS 14.0, *) + public var supportedUTTypes: [UTType] { + supportedUTIs.compactMap { UTType($0) } + } + /// Supported document media types. public let supportedMediaTypes: [MediaType] diff --git a/Sources/Shared/Toolkit/Weak.swift b/Sources/Shared/Toolkit/Weak.swift index e641762b2..362ac82a0 100644 --- a/Sources/Shared/Toolkit/Weak.swift +++ b/Sources/Shared/Toolkit/Weak.swift @@ -11,7 +11,7 @@ import Foundation /// Get the reference by calling `weakVar()`. /// Conveniently, the reference can be reset by setting the `ref` property. @dynamicCallable -public final class Weak { +public class Weak { // Weakly held reference. public weak var ref: T? @@ -23,4 +23,24 @@ public final class Weak { public func dynamicallyCall(withArguments args: [Any]) -> T? { ref } -} \ No newline at end of file +} + +/// Smart pointer passing as a Weak reference but preventing the reference from being lost. +/// Mainly useful for the unit test suite. +public class _Strong: Weak { + + private var strongRef: T? + + public override var ref: T? { + get { super.ref } + set { + super.ref = newValue + strongRef = newValue + } + } + + public override init(_ ref: T? = nil) { + self.strongRef = ref + super.init(ref) + } +} diff --git a/Sources/Streamer/Parser/Audio/Services/AudioLocatorService.swift b/Sources/Streamer/Parser/Audio/Services/AudioLocatorService.swift index 63e1ccb6d..75edb17da 100644 --- a/Sources/Streamer/Parser/Audio/Services/AudioLocatorService.swift +++ b/Sources/Streamer/Parser/Audio/Services/AudioLocatorService.swift @@ -8,39 +8,26 @@ import Foundation import R2Shared /// Locator service for audio publications. -final class AudioLocatorService: LocatorService { - - /// Total duration of the publication. - private let totalDuration: Double? - +final class AudioLocatorService: DefaultLocatorService { + + static func makeFactory() -> (PublicationServiceContext) -> AudioLocatorService { + { context in AudioLocatorService(publication: context.publication) } + } + + private lazy var readingOrder: [Link] = + publication()?.readingOrder ?? [] + /// Duration per reading order index. - private let durations: [Double] - - private let readingOrder: [Link] - - init(readingOrder: [Link]) { - self.durations = readingOrder.map { $0.duration ?? 0 } + private lazy var durations: [Double] = + readingOrder.map { $0.duration ?? 0 } + + /// Total duration of the publication. + private lazy var totalDuration: Double? = { let totalDuration = durations.reduce(0, +) - self.totalDuration = (totalDuration > 0) ? totalDuration : nil - self.readingOrder = readingOrder - } - - func locate(_ locator: Locator) -> Locator? { - if readingOrder.firstIndex(withHREF: locator.href) != nil { - return locator - } - - if let totalProgression = locator.locations.totalProgression, let target = locate(progression: totalProgression) { - return target.copy( - title: locator.title, - text: { $0 = locator.text } - ) - } - - return nil - } - - func locate(progression: Double) -> Locator? { + return (totalDuration > 0) ? totalDuration : nil + }() + + override func locate(progression: Double) -> Locator? { guard let totalDuration = totalDuration else { return nil } @@ -68,11 +55,7 @@ final class AudioLocatorService: LocatorService { ) ) } - - static func makeFactory() -> (PublicationServiceContext) -> AudioLocatorService { - { context in AudioLocatorService(readingOrder: context.manifest.readingOrder) } - } - + /// Finds the reading order item containing the time `position` (in seconds), as well as its /// start time. private func readingOrderItemAtPosition(_ position: Double) -> (link: Link, startPosition: Double)? { diff --git a/Support/Carthage/Readium.xcodeproj/project.pbxproj b/Support/Carthage/Readium.xcodeproj/project.pbxproj index 2458f598c..8b2068588 100644 --- a/Support/Carthage/Readium.xcodeproj/project.pbxproj +++ b/Support/Carthage/Readium.xcodeproj/project.pbxproj @@ -274,6 +274,7 @@ E8CB4E5729E7000FC55FC937 /* CoreServices.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 342D5C0FEE79A2ABEE24A43E /* CoreServices.framework */; }; E9AADF25494C968A44979B66 /* UInt64.swift in Sources */ = {isa = PBXBuildFile; fileRef = 57338C29681D4872D425AB81 /* UInt64.swift */; }; E9EE047B89D0084523F7C888 /* Feed.swift in Sources */ = {isa = PBXBuildFile; fileRef = C5E7CEDF6EA681FE8119791B /* Feed.swift */; }; + EA8C7F894E3BE8D6D954DC47 /* InMemoryPositionsService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 505BF8A630F7C7B96754E333 /* InMemoryPositionsService.swift */; }; EB4D11D2D1A0C64FF0E982C3 /* Optional.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC925E451D875E5F74748EDC /* Optional.swift */; }; EDCA3449EA5683B37D82FEBE /* PDFKit.swift in Sources */ = {isa = PBXBuildFile; fileRef = ABAF1D0444B94E2CDD80087D /* PDFKit.swift */; }; EE951A131E38E316BF7A1129 /* LCPDialogViewController.xib in Resources */ = {isa = PBXBuildFile; fileRef = ED5C6546C24E5E619E4CC9D1 /* LCPDialogViewController.xib */; }; @@ -412,6 +413,7 @@ 4BB5D42EEF0083D833E2A572 /* Publication+OPDS.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Publication+OPDS.swift"; sourceTree = ""; }; 4E564AE6D5137499C81FEBE2 /* TargetAction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TargetAction.swift; sourceTree = ""; }; 500E55D9CA753D6D6AA76D10 /* EPUBLicenseContainer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EPUBLicenseContainer.swift; sourceTree = ""; }; + 505BF8A630F7C7B96754E333 /* InMemoryPositionsService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InMemoryPositionsService.swift; sourceTree = ""; }; 508E0CD4F9F02CC851E6D1E1 /* Publication+EPUB.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Publication+EPUB.swift"; sourceTree = ""; }; 54699BC0E00F327E67908F6A /* Encryption.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Encryption.swift; sourceTree = ""; }; 55BC4119B8937D17ED80B1AB /* ControlFlow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ControlFlow.swift; sourceTree = ""; }; @@ -982,6 +984,7 @@ 5BC52D8F4F854FDA56D10A8E /* Positions */ = { isa = PBXGroup; children = ( + 505BF8A630F7C7B96754E333 /* InMemoryPositionsService.swift */, 01CCE64AE9824DCF6D6413BC /* PerResourcePositionsService.swift */, BC45956B8991A9488F957B06 /* PositionsService.swift */, ); @@ -1882,6 +1885,7 @@ 39FC65D3797EF5069A04F34B /* HTTPFetcher.swift in Sources */, 2B57BE89EFAE517F79A17667 /* HTTPProblemDetails.swift in Sources */, FD13DEAC62A3ED6714841B7A /* HTTPRequest.swift in Sources */, + EA8C7F894E3BE8D6D954DC47 /* InMemoryPositionsService.swift in Sources */, 9C6B7AFB6FB0635EF5B7B71C /* JSON.swift in Sources */, 69150D0B00F5665C3DA0000B /* LazyResource.swift in Sources */, 5C9617AE1B5678A95ABFF1AA /* Link.swift in Sources */, diff --git a/TestApp/.gitignore b/TestApp/.gitignore index 8225e5fbf..3c74969de 100644 --- a/TestApp/.gitignore +++ b/TestApp/.gitignore @@ -1,6 +1,8 @@ TestApp.xcodeproj TestApp.xcworkspace project.yml +# IntelliJ AppCode +.idea Carthage/ Cartfile diff --git a/TestApp/Sources/Library/LibraryViewController.swift b/TestApp/Sources/Library/LibraryViewController.swift index 1514c910b..6c56bfa5b 100644 --- a/TestApp/Sources/Library/LibraryViewController.swift +++ b/TestApp/Sources/Library/LibraryViewController.swift @@ -19,7 +19,7 @@ import R2Streamer import R2Navigator import Kingfisher import ReadiumOPDS - +import UniformTypeIdentifiers protocol LibraryViewControllerFactory { func make() -> LibraryViewController @@ -50,11 +50,6 @@ class LibraryViewController: UIViewController, Loggable { @IBOutlet weak var collectionView: UICollectionView! { didSet { - // The contentInset of collectionVIew might be changed by iOS 9/10. - // This property has been set as false on storyboard. - // In case it's changed by mistake somewhere, set it again here. - self.automaticallyAdjustsScrollViewInsets = false - collectionView.backgroundColor = #colorLiteral(red: 1.0, green: 1.0, blue: 1.0, alpha: 1.0) collectionView.contentInset = UIEdgeInsets(top: 15, left: 20, bottom: 20, right: 20) @@ -155,9 +150,12 @@ class LibraryViewController: UIViewController, Loggable { } private func addBookFromDevice() { - var utis = DocumentTypes.main.supportedUTIs - utis.append(String(kUTTypeText)) - let documentPicker = UIDocumentPickerViewController(documentTypes: utis, in: .import) + var types = DocumentTypes.main.supportedUTTypes + if let type = UTType(String(kUTTypeText)) { + types.append(type) + } + + let documentPicker = UIDocumentPickerViewController(forOpeningContentTypes: types) documentPicker.delegate = self present(documentPicker, animated: true, completion: nil) } @@ -236,9 +234,6 @@ extension LibraryViewController { extension LibraryViewController: UIDocumentPickerDelegate { public func documentPicker(_ controller: UIDocumentPickerViewController, didPickDocumentsAt urls: [URL]) { - guard controller.documentPickerMode == .import else { - return - } importFiles(at: urls) } diff --git a/TestApp/Sources/OPDS/OPDSCatalogSelectorViewController.swift b/TestApp/Sources/OPDS/OPDSCatalogSelectorViewController.swift index 1e9e5d468..6f991589a 100644 --- a/TestApp/Sources/OPDS/OPDSCatalogSelectorViewController.swift +++ b/TestApp/Sources/OPDS/OPDSCatalogSelectorViewController.swift @@ -104,23 +104,30 @@ class OPDSCatalogSelectorViewController: UITableViewController { navigationController?.pushViewController(viewController, animated: true) } - override func tableView(_ tableView: UITableView, editActionsForRowAt indexPath: IndexPath) -> [UITableViewRowAction]? { - + override func tableView(_ tableView: UITableView, trailingSwipeActionsConfigurationForRowAt indexPath: IndexPath) -> UISwipeActionsConfiguration? { // action one - let editAction = UITableViewRowAction(style: .default, title: NSLocalizedString("edit_button", comment: "Edit a OPDS feed button"), handler: { (action, indexPath) in + let editAction = UIContextualAction( + style: .normal, + title: NSLocalizedString("edit_button", comment: "Edit a OPDS feed button") + ) { _, _, completion in self.showEditPopup(feedIndex: indexPath.row) - }) + completion(true) + } editAction.backgroundColor = UIColor.gray // action two - let deleteAction = UITableViewRowAction(style: .default, title: NSLocalizedString("remove_button", comment: "Remove an OPDS feed button"), handler: { (action, indexPath) in + let deleteAction = UIContextualAction( + style: .destructive, + title: NSLocalizedString("remove_button", comment: "Remove an OPDS feed button") + ) { _, _, completion in self.catalogData?.remove(at: indexPath.row) UserDefaults.standard.set(self.catalogData, forKey: self.userDefaultsID) self.tableView.reloadData() - }) + completion(true) + } deleteAction.backgroundColor = UIColor.gray - return [editAction, deleteAction] + return UISwipeActionsConfiguration(actions: [editAction, deleteAction]) } @objc func showAddFeedPopup() { diff --git a/TestApp/Sources/Reader/Common/Outline/OutlineTableView.swift b/TestApp/Sources/Reader/Common/Outline/OutlineTableView.swift index e0c08d1ea..fc5276ac1 100644 --- a/TestApp/Sources/Reader/Common/Outline/OutlineTableView.swift +++ b/TestApp/Sources/Reader/Common/Outline/OutlineTableView.swift @@ -19,27 +19,28 @@ enum OutlineSection: Int { } struct OutlineTableView: View { + private let publication: Publication @ObservedObject private var bookmarksModel: BookmarksViewModel @ObservedObject private var highlightsModel: HighlightsViewModel @State private var selectedSection: OutlineSection = .tableOfContents // Outlines (list of links) to display for each section. private var outlines: [OutlineSection: [(level: Int, link: R2Shared.Link)]] = [:] - + init(publication: Publication, bookId: Book.Id, bookmarkRepository: BookmarkRepository, highlightRepository: HighlightRepository) { - + self.publication = publication + self.bookmarksModel = BookmarksViewModel(bookId: bookId, repository: bookmarkRepository) + self.highlightsModel = HighlightsViewModel(bookId: bookId, repository: highlightRepository) + func flatten(_ links: [R2Shared.Link], level: Int = 0) -> [(level: Int, link: R2Shared.Link)] { return links.flatMap { [(level, $0)] + flatten($0.children, level: level + 1) } } - outlines = [ + self.outlines = [ .tableOfContents: flatten(publication.tableOfContents), .landmarks: flatten(publication.landmarks), .pageList: flatten(publication.pageList) ] - - bookmarksModel = BookmarksViewModel(bookId: bookId, repository: bookmarkRepository) - highlightsModel = HighlightsViewModel(bookId: bookId, repository: highlightRepository) } var body: some View { @@ -55,7 +56,9 @@ struct OutlineTableView: View { .frame(maxWidth: .infinity, alignment: .leading) .contentShape(Rectangle()) .onTapGesture { - locatorSubject.send(Locator(link: item.link)) + if let locator = publication.locate(item.link) { + locatorSubject.send(locator) + } } } } else { diff --git a/Tests/SharedTests/Publication/LocatorTests.swift b/Tests/SharedTests/Publication/LocatorTests.swift index edfb0afca..3e3f1b5ed 100644 --- a/Tests/SharedTests/Publication/LocatorTests.swift +++ b/Tests/SharedTests/Publication/LocatorTests.swift @@ -75,47 +75,6 @@ class LocatorTests: XCTestCase { XCTAssertEqual([Locator](json: nil), []) } - func testMakeFromFullLink() { - XCTAssertEqual( - Locator(link: Link( - href: "http://locator", - type: "text/html", - title: "Link title" - )), - Locator( - href: "http://locator", - type: "text/html", - title: "Link title" - ) - ) - } - - func testMakeFromMinimalLink() { - XCTAssertEqual( - Locator(link: Link( - href: "http://locator" - )), - Locator( - href: "http://locator", - type: "", - title: nil - ) - ) - } - - func testMakeFromLinkWithFragment() { - XCTAssertEqual( - Locator(link: Link( - href: "http://locator#page=42" - )), - Locator( - href: "http://locator", - type: "", - locations: .init(fragments: ["page=42"]) - ) - ) - } - func testGetMinimalJSON() { AssertJSONEqual( Locator( diff --git a/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift b/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift index 9fd681d1f..a0ed65605 100644 --- a/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift +++ b/Tests/SharedTests/Publication/Services/Locator/DefaultLocatorServiceTests.swift @@ -113,10 +113,88 @@ class DefaultLocatorServiceTests: XCTestCase { XCTAssertNil(service.locate(progression: 0.5)) } - func makeService(readingOrder: [Link] = [], positions: [[Locator]] = []) -> DefaultLocatorService { - DefaultLocatorService(readingOrder: readingOrder, positionsByReadingOrder: { positions }) + func testFromMinimalLink() { + let service = makeService(readingOrder: [ + Link(href: "/href", type: "text/html", title: "Resource") + ]) + + XCTAssertEqual( + service.locate(Link(href: "/href")), + Locator(href: "/href", type: "text/html", title: "Resource", locations: Locator.Locations(progression: 0.0)) + ) + } + + func testFromLinkInReadingOrderResourcesOrLinks() { + let service = makeService( + links: [Link(href: "/href3", type: "text/html")], + readingOrder: [Link(href: "/href1", type: "text/html")], + resources: [Link(href: "/href2", type: "text/html")] + ) + + XCTAssertEqual( + service.locate(Link(href: "/href1")), + Locator(href: "/href1", type: "text/html", locations: Locator.Locations(progression: 0.0)) + ) + + XCTAssertEqual( + service.locate(Link(href: "/href2")), + Locator(href: "/href2", type: "text/html", locations: Locator.Locations(progression: 0.0)) + ) + + XCTAssertEqual( + service.locate(Link(href: "/href3")), + Locator(href: "/href3", type: "text/html", locations: Locator.Locations(progression: 0.0)) + ) + } + + func testFromLinkWithFragment() { + let service = makeService(readingOrder: [ + Link(href: "/href", type: "text/html", title: "Resource") + ]) + + XCTAssertEqual( + service.locate(Link(href: "/href#page=42", type: "text/xml", title: "My link")), + Locator(href: "/href", type: "text/html", title: "Resource", locations: Locator.Locations(fragments: ["page=42"])) + ) + } + + func testTitleFallbackFromLink() { + let service = makeService(readingOrder: [ + Link(href: "/href", type: "text/html") + ]) + + XCTAssertEqual( + service.locate(Link(href: "/href", title: "My link")), + Locator(href: "/href", type: "text/html", title: "My link", locations: Locator.Locations(progression: 0.0)) + ) + } + + func testFromLinkNotFound() { + let service = makeService(readingOrder: [ + Link(href: "/href", type: "text/html") + ]) + + XCTAssertNil(service.locate(Link(href: "notfound"))) } + func makeService( + links: [Link] = [], + readingOrder: [Link] = [], + resources: [Link] = [], + positions: [[Locator]] = [] + ) -> DefaultLocatorService { + DefaultLocatorService(publication: _Strong(Publication( + manifest: Manifest( + metadata: Metadata(title: ""), + links: links, + readingOrder: readingOrder, + resources: resources + ), + servicesBuilder: PublicationServicesBuilder( + positions: InMemoryPositionsService.makeFactory(positionsByReadingOrder: positions) + ) + ))) + } } private let positionsFixture: [[Locator]] = [ diff --git a/Tests/StreamerTests/Parser/Audio/Services/AudioLocatorServiceTests.swift b/Tests/StreamerTests/Parser/Audio/Services/AudioLocatorServiceTests.swift index 3f9825973..93dc64dcc 100644 --- a/Tests/StreamerTests/Parser/Audio/Services/AudioLocatorServiceTests.swift +++ b/Tests/StreamerTests/Parser/Audio/Services/AudioLocatorServiceTests.swift @@ -165,7 +165,9 @@ class AudioLocatorServiceTests: XCTestCase { private func makeService(readingOrder: [Link]) -> AudioLocatorService { AudioLocatorService( - readingOrder: readingOrder + publication: _Strong(Publication( + manifest: Manifest(metadata: Metadata(title: ""), readingOrder: readingOrder) + )) ) } From 15b9546bc8cd2eb3a40a3a9b7d911dfb50dd8b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Tue, 10 May 2022 14:46:04 +0200 Subject: [PATCH 20/46] Go to currently playing word --- Sources/Navigator/TTS/TTSEngine.swift | 16 ++- Sources/Shared/Publication/Locator.swift | 24 ++++- .../Reader/Common/TTS/TTSViewModel.swift | 20 +++- .../Publication/LocatorTests.swift | 97 +++++++++++++++++-- 4 files changed, 142 insertions(+), 15 deletions(-) diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index d90f2a776..60844428d 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -83,20 +83,26 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg } public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { - guard let utterance = utterance as? AVUtterance else { + guard let utterance = (utterance as? AVUtterance)?.utterance else { return } - delegate?.ttsEngine(self, didFinish: utterance.utterance) + delegate?.ttsEngine(self, didFinish: utterance) } public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance avUtterance: AVSpeechUtterance) { guard let delegate = delegate, - let range = Range(characterRange) - else { + let utterance = (avUtterance as? AVUtterance)?.utterance, + let highlight = utterance.locator.text.highlight, + let range = Range(characterRange, in: highlight) + else { return } -// controller?.notifySpeakingRange(range) + + let rangeLocator = utterance.locator.copy( + text: { text in text = text[range] } + ) + delegate.ttsEngine(self, willSpeakRangeAt: rangeLocator, of: utterance) } private class AVUtterance: AVSpeechUtterance { diff --git a/Sources/Shared/Publication/Locator.swift b/Sources/Shared/Publication/Locator.swift index 4d57543cb..64085ee84 100644 --- a/Sources/Shared/Publication/Locator.swift +++ b/Sources/Shared/Publication/Locator.swift @@ -268,7 +268,29 @@ public struct Locator: Hashable, CustomStringConvertible, Loggable { highlight: highlight?.coalescingWhitespaces() ) } - + + /// Returns a copy of this text after highlighting a sub-range in the `highlight` property. + /// + /// The bounds of the range must be valid indices of the `highlight` property. + public subscript(range: R) -> Text where R : RangeExpression, R.Bound == String.Index { + guard let highlight = highlight else { + preconditionFailure("highlight is nil") + } + + let range = range.relative(to: highlight) + var before = before ?? "" + var after = after ?? "" + let newHighlight = highlight[range] + before = before + highlight[..() private var subscriptions: Set = [] init?(navigator: Navigator, publication: Publication) { @@ -39,6 +41,19 @@ final class TTSViewModel: ObservableObject, Loggable { tts.config = $0 } .store(in: &subscriptions) + + var isMoving = false + playingRangeLocatorSubject + .throttle(for: 1, scheduler: RunLoop.main, latest: true) + .sink { [unowned self] locator in + guard !isMoving else { + return + } + isMoving = navigator.go(to: locator) { + isMoving = false + } + } + .store(in: &subscriptions) } var defaultRate: Double { tts.defaultRate } @@ -100,7 +115,10 @@ extension TTSViewModel: TTSControllerDelegate { } public func ttsController(_ ttsController: TTSController, willStartSpeaking utterance: TTSUtterance) { - navigator.go(to: utterance.locator) highlight(utterance) } + + public func ttsController(_ ttsController: TTSController, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) { + playingRangeLocatorSubject.send(locator) + } } \ No newline at end of file diff --git a/Tests/SharedTests/Publication/LocatorTests.swift b/Tests/SharedTests/Publication/LocatorTests.swift index 3e3f1b5ed..7719775af 100644 --- a/Tests/SharedTests/Publication/LocatorTests.swift +++ b/Tests/SharedTests/Publication/LocatorTests.swift @@ -1,12 +1,7 @@ // -// LocatorTests.swift -// r2-shared-swiftTests -// -// Created by Mickaël Menu on 20.03.19. -// -// Copyright 2019 Readium Foundation. All rights reserved. -// Use of this source code is governed by a BSD-style license which is detailed -// in the LICENSE file present in the project repository where this source code is maintained. +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. // import XCTest @@ -576,4 +571,90 @@ class LocatorCollectionTests: XCTestCase { ) ) } + + func testGetRangeOfText() { + let highlight = "highlight" + + XCTAssertEqual( + Locator.Text( + after: "after", + before: "before", + highlight: highlight + )[highlight.range(of: "ghl")!], + Locator.Text( + after: "ightafter", + before: "beforehi", + highlight: "ghl" + ) + ) + + XCTAssertEqual( + Locator.Text( + after: "after", + before: "before", + highlight: highlight + )[highlight.range(of: "hig")!], + Locator.Text( + after: "hlightafter", + before: "before", + highlight: "hig" + ) + ) + + XCTAssertEqual( + Locator.Text( + after: "after", + before: "before", + highlight: highlight + )[highlight.range(of: "light")!], + Locator.Text( + after: "after", + before: "beforehigh", + highlight: "light" + ) + ) + } + + func testGetRangeOfTextWithNilProperties() { + let highlight = "highlight" + + XCTAssertEqual( + Locator.Text( + after: nil, + before: nil, + highlight: highlight + )[highlight.range(of: "ghl")!], + Locator.Text( + after: "ight", + before: "hi", + highlight: "ghl" + ) + ) + + XCTAssertEqual( + Locator.Text( + after: "after", + before: nil, + highlight: highlight + )[highlight.range(of: "hig")!], + Locator.Text( + after: "hlightafter", + before: nil, + highlight: "hig" + ) + ) + + XCTAssertEqual( + Locator.Text( + after: nil, + before: "before", + highlight: highlight + )[highlight.range(of: "light")!], + Locator.Text( + after: nil, + before: "beforehigh", + highlight: "light" + ) + ) + } } From 6fdc0a57c868199e47f7bc02cc3ae2093d9110ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 11 May 2022 10:19:01 +0200 Subject: [PATCH 21/46] Refactor TTS engine and add `Language` --- Sources/Navigator/TTS/AVTTSEngine.swift | 153 ++++++++++++++++++ Sources/Navigator/TTS/TTSController.swift | 66 +++----- Sources/Navigator/TTS/TTSEngine.swift | 139 +++++----------- Sources/Shared/Publication/Metadata.swift | 1 + .../Services/Content/ContentIterator.swift | 4 +- .../Content/HTMLResourceContentIterator.swift | 4 +- .../Services/Search/SearchService.swift | 6 +- .../Services/Search/StringSearchService.swift | 36 ++--- Sources/Shared/Toolkit/Language.swift | 42 +++++ .../Toolkit/Tokenizer/ContentTokenizer.swift | 2 +- .../Toolkit/Tokenizer/TextTokenizer.swift | 6 +- .../Sources/Reader/Common/TTS/TTSView.swift | 37 ++++- .../Reader/Common/TTS/TTSViewModel.swift | 11 +- 13 files changed, 336 insertions(+), 171 deletions(-) create mode 100644 Sources/Navigator/TTS/AVTTSEngine.swift create mode 100644 Sources/Shared/Toolkit/Language.swift diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift new file mode 100644 index 000000000..dc171b96c --- /dev/null +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -0,0 +1,153 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import AVFoundation +import Foundation +import R2Shared + +public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Loggable { + + /// Range of valid values for an AVUtterance rate. + /// + /// > The speech rate is a decimal representation within the range of `AVSpeechUtteranceMinimumSpeechRate` and + /// > `AVSpeechUtteranceMaximumSpeechRate`. Lower values correspond to slower speech, and higher values correspond to + /// > faster speech. The default value is `AVSpeechUtteranceDefaultSpeechRate`. + /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619708-rate + private let avRateRange = + Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate) + + /// Range of valid values for an AVUtterance pitch. + /// + /// > Before enqueuing the utterance, set this property to a value within the range of 0.5 for lower pitch to 2.0 for + /// > higher pitch. The default value is 1.0. + /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier + private let avPitchRange = 0.5...2.0 + + public let defaultConfig: TTSConfiguration + public var config: TTSConfiguration + + public weak var delegate: TTSEngineDelegate? + + private let synthesizer = AVSpeechSynthesizer() + + public override init() { + let config = TTSConfiguration( + defaultLanguage: Language(code: .bcp47(AVSpeechSynthesisVoice.currentLanguageCode())), + rate: avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)), + pitch: avPitchRange.percentageForValue(1.0) + ) + + self.defaultConfig = config + self.config = config + + super.init() + synthesizer.delegate = self + } + + public lazy var availableVoices: [TTSVoice] = + AVSpeechSynthesisVoice.speechVoices().map { v in + TTSVoice( + identifier: v.identifier, + language: Language(code: .bcp47(v.language)), + name: v.name, + gender: .init(voice: v), + quality: .init(voice: v) + ) + } + + public func speak(_ utterance: TTSUtterance) { + synthesizer.stopSpeaking(at: .immediate) + synthesizer.speak(avUtterance(from: utterance)) + } + + public func stop() { + synthesizer.stopSpeaking(at: .immediate) + } + + private func avUtterance(from utterance: TTSUtterance) -> AVSpeechUtterance { + let avUtterance = AVUtterance(utterance: utterance) + avUtterance.rate = Float(avRateRange.valueForPercentage(config.rate)) + avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(config.pitch)) + avUtterance.preUtteranceDelay = utterance.delay + avUtterance.postUtteranceDelay = config.delay + avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language ?? config.defaultLanguage) + return avUtterance + } + + public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + guard let utterance = (utterance as? AVUtterance)?.utterance else { + return + } + delegate?.ttsEngine(self, didFinish: utterance) + } + + public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance avUtterance: AVSpeechUtterance) { + guard + let delegate = delegate, + let utterance = (avUtterance as? AVUtterance)?.utterance, + let highlight = utterance.locator.text.highlight, + let range = Range(characterRange, in: highlight) + else { + return + } + + let rangeLocator = utterance.locator.copy( + text: { text in text = text[range] } + ) + delegate.ttsEngine(self, willSpeakRangeAt: rangeLocator, of: utterance) + } + + private class AVUtterance: AVSpeechUtterance { + let utterance: TTSUtterance + + init(utterance: TTSUtterance) { + self.utterance = utterance + super.init(string: utterance.text) + } + + required init?(coder: NSCoder) { + fatalError("Not supported") + } + } +} + +private extension TTSVoice.Gender { + init(voice: AVSpeechSynthesisVoice) { + if #available(iOS 13.0, *) { + switch voice.gender { + case .unspecified: + self = .unspecified + case .male: + self = .male + case .female: + self = .female + @unknown default: + self = .unspecified + } + } else { + self = .unspecified + } + } +} + +private extension TTSVoice.Quality { + init?(voice: AVSpeechSynthesisVoice) { + switch voice.quality { + case .default: + self = .medium + case .enhanced: + self = .high + @unknown default: + return nil + } + } +} + +private extension AVSpeechSynthesisVoice { + convenience init?(language: Language) { + self.init(language: language.code.bcp47) + } +} \ No newline at end of file diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index 8fd9fe583..5b9857232 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -25,38 +25,30 @@ public extension TTSControllerDelegate { public struct TTSUtterance: Equatable { public let text: String public let locator: Locator - public let language: String? - public let pitch: Double? - public let rate: Double? - public let postDelay: TimeInterval + public let language: Language? + public let delay: TimeInterval } public class TTSController: Loggable, TTSEngineDelegate { - public typealias TokenizerFactory = (_ language: String?) -> ContentTokenizer - - public struct Configuration { - public var defaultLanguage: String? - public var rate: Double - public var pitch: Double - - public init( - defaultLanguage: String?, - rate: Double, - pitch: Double - ) { - self.defaultLanguage = defaultLanguage - self.rate = rate - self.pitch = pitch - } - } + public typealias TokenizerFactory = (_ language: Language?) -> ContentTokenizer public static func canPlay(_ publication: Publication) -> Bool { publication.isContentIterable } - public let defaultRate: Double - public let defaultPitch: Double - public var config: Configuration + public var defaultConfig: TTSConfiguration { + engine.defaultConfig + } + + public var config: TTSConfiguration { + get { engine.config } + set { engine.config = newValue } + } + + public var availableVoices: [TTSVoice] { + engine.availableVoices + } + public weak var delegate: TTSControllerDelegate? private let publication: Publication @@ -67,14 +59,11 @@ public class TTSController: Loggable, TTSEngineDelegate { public init( publication: Publication, engine: TTSEngine = AVTTSEngine(), - tokenizerFactory: TokenizerFactory? = nil, + makeTokenizer: TokenizerFactory? = nil, delegate: TTSControllerDelegate? = nil ) { precondition(publication.isContentIterable, "The Publication must be iterable to be used with TTSController") - self.defaultRate = engine.defaultRate ?? 0.5 - self.defaultPitch = engine.defaultPitch ?? 0.5 - self.config = Configuration(defaultLanguage: nil, rate: defaultRate, pitch: defaultPitch) self.delegate = delegate self.publication = publication self.engine = engine @@ -85,6 +74,9 @@ public class TTSController: Loggable, TTSEngineDelegate { ) } + if let language = publication.metadata.languages.first { + engine.config.defaultLanguage = Language(code: .bcp47(language)) + } engine.delegate = self } @@ -226,7 +218,7 @@ public class TTSController: Loggable, TTSEngineDelegate { return false } - utterances = tokenize(content, with: makeTokenizer(defaultLanguage)) + utterances = tokenize(content, with: makeTokenizer(nil)) .flatMap { utterances(from: $0) } guard !utterances.isEmpty else { @@ -255,32 +247,24 @@ public class TTSController: Loggable, TTSEngineDelegate { text: span.text, locator: span.locator, language: span.language, - postDelay: (offset == spans.count - 1) ? 0.4 : 0 + delay: (offset == 0) ? 0.4 : 0 ) } } } - private func utterance(text: String, locator: Locator, language: String? = nil, postDelay: TimeInterval = 0) -> TTSUtterance? { + private func utterance(text: String, locator: Locator, language: Language? = nil, delay: TimeInterval = 0) -> TTSUtterance? { guard text.contains(where: { $0.isLetter || $0.isNumber }) else { return nil } return TTSUtterance( text: text, locator: locator, - language: language ?? defaultLanguage, - pitch: config.pitch, - rate: config.rate, - postDelay: postDelay + language: language, + delay: delay ) } - private var defaultLanguage: String? { - config.defaultLanguage - ?? publication.metadata.languages.first - ?? engine.defaultLanguage - } - private func tokenize(_ content: Content, with tokenizer: ContentTokenizer) -> [Content] { do { return try tokenizer(content) diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index 60844428d..6ba57a8d4 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -4,117 +4,66 @@ // available in the top-level LICENSE file of the project. // -import AVFoundation import Foundation import R2Shared -public protocol TTSEngineDelegate: AnyObject { - func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) - func ttsEngine(_ engine: TTSEngine, didFinish utterance: TTSUtterance) -} - public protocol TTSEngine: AnyObject { - var defaultLanguage: String? { get } - var defaultRate: Double? { get } - var defaultPitch: Double? { get } + var defaultConfig: TTSConfiguration { get } + var config: TTSConfiguration { get set } var delegate: TTSEngineDelegate? { get set } + var availableVoices: [TTSVoice] { get } func speak(_ utterance: TTSUtterance) func stop() } -public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Loggable { - - public var defaultLanguage: String? { - AVSpeechSynthesisVoice.currentLanguageCode() - } - - public lazy var defaultRate: Double? = - avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)) - - /// Range of valid values for an AVUtterance rate. - /// - /// > The speech rate is a decimal representation within the range of `AVSpeechUtteranceMinimumSpeechRate` and - /// > `AVSpeechUtteranceMaximumSpeechRate`. Lower values correspond to slower speech, and higher values correspond to - /// > faster speech. The default value is `AVSpeechUtteranceDefaultSpeechRate`. - /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619708-rate - private let avRateRange = - Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate) - - public lazy var defaultPitch: Double? = - avPitchRange.percentageForValue(1.0) - - /// Range of valid values for an AVUtterance pitch. - /// - /// > Before enqueing the utterance, set this property to a value within the range of 0.5 for lower pitch to 2.0 for - /// > higher pitch. The default value is 1.0. - /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier - private let avPitchRange = 0.5...2.0 - - public weak var delegate: TTSEngineDelegate? - - private let synthesizer = AVSpeechSynthesizer() - - public override init() { - super.init() - synthesizer.delegate = self - } - - public func speak(_ utterance: TTSUtterance) { - synthesizer.stopSpeaking(at: .immediate) - synthesizer.speak(avUtterance(from: utterance)) - } - - public func stop() { - synthesizer.stopSpeaking(at: .immediate) - } +public protocol TTSEngineDelegate: AnyObject { + func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) + func ttsEngine(_ engine: TTSEngine, didFinish utterance: TTSUtterance) +} - private func avUtterance(from utterance: TTSUtterance) -> AVSpeechUtterance { - let avUtterance = AVUtterance(utterance: utterance) - if let rate = utterance.rate ?? defaultRate { - avUtterance.rate = Float(avRateRange.valueForPercentage(rate)) - } - if let pitch = utterance.pitch ?? defaultPitch { - avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(pitch)) - } - avUtterance.postUtteranceDelay = utterance.postDelay - avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language) - return avUtterance +public struct TTSConfiguration { + public var defaultLanguage: Language + public var rate: Double + public var pitch: Double + public var voice: TTSVoice? + public var delay: TimeInterval + + public init( + defaultLanguage: Language, + rate: Double = 0.5, + pitch: Double = 0.5, + voice: TTSVoice? = nil, + delay: TimeInterval = 0 + ) { + self.defaultLanguage = defaultLanguage + self.rate = rate + self.pitch = pitch + self.voice = voice + self.delay = delay } +} - public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { - guard let utterance = (utterance as? AVUtterance)?.utterance else { - return - } - delegate?.ttsEngine(self, didFinish: utterance) +public struct TTSVoice { + public enum Gender { + case female, male, unspecified } - public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance avUtterance: AVSpeechUtterance) { - guard - let delegate = delegate, - let utterance = (avUtterance as? AVUtterance)?.utterance, - let highlight = utterance.locator.text.highlight, - let range = Range(characterRange, in: highlight) - else { - return - } - - let rangeLocator = utterance.locator.copy( - text: { text in text = text[range] } - ) - delegate.ttsEngine(self, willSpeakRangeAt: rangeLocator, of: utterance) + public enum Quality { + case low, medium, high } - private class AVUtterance: AVSpeechUtterance { - let utterance: TTSUtterance - - init(utterance: TTSUtterance) { - self.utterance = utterance - super.init(string: utterance.text) - } - - required init?(coder: NSCoder) { - fatalError("Not supported") - } + public let identifier: String + public let language: Language + public let name: String + public let gender: Gender + public let quality: Quality? + + public init(identifier: String, language: Language, name: String, gender: Gender, quality: Quality?) { + self.identifier = identifier + self.language = language + self.name = name + self.gender = gender + self.quality = quality } } diff --git a/Sources/Shared/Publication/Metadata.swift b/Sources/Shared/Publication/Metadata.swift index fd398b8b2..7c502f356 100644 --- a/Sources/Shared/Publication/Metadata.swift +++ b/Sources/Shared/Publication/Metadata.swift @@ -30,6 +30,7 @@ public struct Metadata: Hashable, Loggable, WarningLogger { public let modified: Date? public let published: Date? + // FIXME: Use `Language` instead of raw String public let languages: [String] // BCP 47 tag public let sortAs: String? public let subjects: [Subject] diff --git a/Sources/Shared/Publication/Services/Content/ContentIterator.swift b/Sources/Shared/Publication/Services/Content/ContentIterator.swift index 584c29c79..320a4fec2 100644 --- a/Sources/Shared/Publication/Services/Content/ContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/ContentIterator.swift @@ -41,10 +41,10 @@ public struct Content: Equatable { public struct TextSpan: Equatable { public let locator: Locator - public let language: String? + public let language: Language? public let text: String - public init(locator: Locator, language: String?, text: String) { + public init(locator: Locator, language: Language?, text: String) { self.locator = locator self.language = language self.text = text diff --git a/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift b/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift index c3b58c7e6..3b104a8e4 100644 --- a/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift @@ -83,7 +83,7 @@ public class HTMLResourceContentIterator : ContentIterator { private var wholeRawTextAcc: String = "" private var elementRawTextAcc: String = "" private var rawTextAcc: String = "" - private var currentLanguage: String? + private var currentLanguage: Language? private var currentCSSSelector: String? private var ignoredNode: Node? @@ -145,7 +145,7 @@ public class HTMLResourceContentIterator : ContentIterator { } if let node = node as? TextNode { - let language = try node.language() + let language = try node.language().map { Language(code: .bcp47($0)) } if (currentLanguage != language) { flushSpan() currentLanguage = language diff --git a/Sources/Shared/Publication/Services/Search/SearchService.swift b/Sources/Shared/Publication/Services/Search/SearchService.swift index 603faeb8f..06a964bfd 100644 --- a/Sources/Shared/Publication/Services/Search/SearchService.swift +++ b/Sources/Shared/Publication/Services/Search/SearchService.swift @@ -96,8 +96,8 @@ public struct SearchOptions: Hashable { /// Matches results exactly as stated in the query terms, taking into account stop words, order and spelling. public var exact: Bool? - /// BCP 47 language code overriding the publication's language. - public var language: String? + /// Language overriding the publication's language. + public var language: Language? /// The search string is treated as a regular expression. /// The particular flavor of regex depends on the service. @@ -118,7 +118,7 @@ public struct SearchOptions: Hashable { diacriticSensitive: Bool? = nil, wholeWord: Bool? = nil, exact: Bool? = nil, - language: String? = nil, + language: Language? = nil, regularExpression: Bool? = nil, otherOptions: [String: String] = [:] ) { diff --git a/Sources/Shared/Publication/Services/Search/StringSearchService.swift b/Sources/Shared/Publication/Services/Search/StringSearchService.swift index cf7674a21..7bff27584 100644 --- a/Sources/Shared/Publication/Services/Search/StringSearchService.swift +++ b/Sources/Shared/Publication/Services/Search/StringSearchService.swift @@ -25,7 +25,7 @@ public class _StringSearchService: _SearchService { return { context in _StringSearchService( publication: context.publication, - language: context.manifest.metadata.languages.first, + language: context.manifest.metadata.languages.first.map { Language(code: .bcp47($0)) }, snippetLength: snippetLength, searchAlgorithm: searchAlgorithm, extractorFactory: extractorFactory @@ -36,27 +36,27 @@ public class _StringSearchService: _SearchService { public let options: SearchOptions private let publication: Weak - private let locale: Locale? + private let language: Language? private let snippetLength: Int private let searchAlgorithm: StringSearchAlgorithm private let extractorFactory: _ResourceContentExtractorFactory - public init(publication: Weak, language: String?, snippetLength: Int, searchAlgorithm: StringSearchAlgorithm, extractorFactory: _ResourceContentExtractorFactory) { + public init(publication: Weak, language: Language?, snippetLength: Int, searchAlgorithm: StringSearchAlgorithm, extractorFactory: _ResourceContentExtractorFactory) { self.publication = publication - self.locale = language.map { Locale(identifier: $0) } + self.language = language self.snippetLength = snippetLength self.searchAlgorithm = searchAlgorithm self.extractorFactory = extractorFactory var options = searchAlgorithm.options - options.language = locale?.languageCode ?? Locale.current.languageCode ?? "en" + options.language = language ?? Language.current self.options = options } public func search(query: String, options: SearchOptions?, completion: @escaping (SearchResult) -> ()) -> Cancellable { let cancellable = CancellableObject() - DispatchQueue.main.async(unlessCancelled: cancellable) { + DispatchQueue.main.async(unlessCancelled: cancellable) { [self] in guard let publication = self.publication() else { completion(.failure(.cancelled)) return @@ -64,10 +64,10 @@ public class _StringSearchService: _SearchService { completion(.success(Iterator( publication: publication, - locale: self.locale, - snippetLength: self.snippetLength, - searchAlgorithm: self.searchAlgorithm, - extractorFactory: self.extractorFactory, + language: language, + snippetLength: snippetLength, + searchAlgorithm: searchAlgorithm, + extractorFactory: extractorFactory, query: query, options: options ))) @@ -81,7 +81,7 @@ public class _StringSearchService: _SearchService { private(set) var resultCount: Int? = 0 private let publication: Publication - private let locale: Locale? + private let language: Language? private let snippetLength: Int private let searchAlgorithm: StringSearchAlgorithm private let extractorFactory: _ResourceContentExtractorFactory @@ -90,7 +90,7 @@ public class _StringSearchService: _SearchService { fileprivate init( publication: Publication, - locale: Locale?, + language: Language?, snippetLength: Int, searchAlgorithm: StringSearchAlgorithm, extractorFactory: _ResourceContentExtractorFactory, @@ -98,7 +98,7 @@ public class _StringSearchService: _SearchService { options: SearchOptions? ) { self.publication = publication - self.locale = locale + self.language = language self.snippetLength = snippetLength self.searchAlgorithm = searchAlgorithm self.extractorFactory = extractorFactory @@ -168,9 +168,9 @@ public class _StringSearchService: _SearchService { var locators: [Locator] = [] - let currentLocale = options.language.map { Locale(identifier: $0) } ?? locale + let currentLanguage = options.language ?? language - for range in searchAlgorithm.findRanges(of: query, options: options, in: text, locale: currentLocale, cancellable: cancellable) { + for range in searchAlgorithm.findRanges(of: query, options: options, in: text, language: currentLanguage, cancellable: cancellable) { guard !cancellable.isCancelled else { return locators } @@ -245,7 +245,7 @@ public protocol StringSearchAlgorithm { /// Finds all the ranges of occurrences of the given `query` in the `text`. /// /// Implementers should check `cancellable.isCancelled` frequently to abort the search if needed. - func findRanges(of query: String, options: SearchOptions, in text: String, locale: Locale?, cancellable: CancellableObject) -> [Range] + func findRanges(of query: String, options: SearchOptions, in text: String, language: Language?, cancellable: CancellableObject) -> [Range] } /// A basic `StringSearchAlgorithm` using the native `String.range(of:)` APIs. @@ -260,7 +260,7 @@ public class BasicStringSearchAlgorithm: StringSearchAlgorithm { public init() {} - public func findRanges(of query: String, options: SearchOptions, in text: String, locale: Locale?, cancellable: CancellableObject) -> [Range] { + public func findRanges(of query: String, options: SearchOptions, in text: String, language: Language?, cancellable: CancellableObject) -> [Range] { var compareOptions: NSString.CompareOptions = [] if options.regularExpression ?? false { compareOptions.insert(.regularExpression) @@ -280,7 +280,7 @@ public class BasicStringSearchAlgorithm: StringSearchAlgorithm { while !cancellable.isCancelled, index < text.endIndex, - let range = text.range(of: query, options: compareOptions, range: index.. /// A `ContentTokenizer` using the default `TextTokenizer` to split the text of the `Content` by `unit`. -public func makeTextContentTokenizer(unit: TextUnit, language: String?) -> ContentTokenizer { +public func makeTextContentTokenizer(unit: TextUnit, language: Language?) -> ContentTokenizer { makeTextContentTokenizer(with: makeDefaultTextTokenizer(unit: unit, language: language)) } diff --git a/Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift index 3ace0305e..0b880d4eb 100644 --- a/Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift +++ b/Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift @@ -20,7 +20,7 @@ public enum TextTokenizerError: Error { } /// A default cluster `Tokenizer` taking advantage of the best capabilities of each iOS version. -public func makeDefaultTextTokenizer(unit: TextUnit, language: String? = nil) -> TextTokenizer { +public func makeDefaultTextTokenizer(unit: TextUnit, language: Language? = nil) -> TextTokenizer { if #available(iOS 12.0, *) { return makeNLTextTokenizer(unit: unit, language: language) } else if #available(iOS 11.0, *) { @@ -35,9 +35,9 @@ public func makeDefaultTextTokenizer(unit: TextUnit, language: String? = nil) -> /// A text `Tokenizer` using iOS 12+'s NaturalLanguage framework. @available(iOS 12.0, *) -public func makeNLTextTokenizer(unit: TextUnit, language: String? = nil) -> TextTokenizer { +public func makeNLTextTokenizer(unit: TextUnit, language: Language? = nil) -> TextTokenizer { let unit = unit.nlUnit - let language = language.map { NLLanguage($0) } + let language = language.map { NLLanguage($0.code.bcp47) } func tokenize(_ text: String) throws -> [Range] { let tokenizer = NLTokenizer(unit: unit) diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index e23c1b702..e1b66b4f4 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -74,13 +74,20 @@ struct TTSSettings: View { ConfigStepper( caption: "Rate", for: \.rate, - step: viewModel.defaultRate / 10 + step: viewModel.defaultConfig.rate / 10 ) ConfigStepper( caption: "Pitch", for: \.pitch, - step: viewModel.defaultPitch / 4 + step: viewModel.defaultConfig.pitch / 4 + ) + + ConfigPicker( + caption: "Language", + for: \.defaultLanguage, + choices: viewModel.availableLanguages, + choiceLabel: { $0.localizedName } ) } } @@ -89,7 +96,7 @@ struct TTSSettings: View { @ViewBuilder func ConfigStepper( caption: String, - for keyPath: WritableKeyPath, + for keyPath: WritableKeyPath, step: Double ) -> some View { Stepper( @@ -104,4 +111,28 @@ struct TTSSettings: View { Text(String.localizedPercentage(viewModel.config[keyPath: keyPath])).font(.footnote) } } + + @ViewBuilder func ConfigPicker( + caption: String, + for keyPath: WritableKeyPath, + choices: [T], + choiceLabel: @escaping (T) -> String + ) -> some View { + HStack { + Text(caption) + Spacer() + + Picker(caption, + selection: Binding( + get: { viewModel.config[keyPath: keyPath] }, + set: { viewModel.config[keyPath: keyPath] = $0 } + ) + ) { + ForEach(choices, id: \.self) { + Text(choiceLabel($0)) + } + } + .pickerStyle(.menu) + } + } } diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index ff0b52705..b51e6a8d3 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -16,7 +16,7 @@ final class TTSViewModel: ObservableObject, Loggable { } @Published private(set) var state: State = .stopped - @Published var config: TTSController.Configuration + @Published var config: TTSConfiguration private let tts: TTSController private let navigator: Navigator @@ -56,8 +56,13 @@ final class TTSViewModel: ObservableObject, Loggable { .store(in: &subscriptions) } - var defaultRate: Double { tts.defaultRate } - var defaultPitch: Double { tts.defaultPitch } + var defaultConfig: TTSConfiguration { tts.defaultConfig } + + lazy var availableLanguages: [Language] = + tts.availableVoices + .map { $0.language } + .removingDuplicates() + .sorted { $0.localizedName < $1.localizedName } @objc func play() { navigator.findLocationOfFirstVisibleContent { [self] locator in From 6d85d819e35a4df4dd335d14e938a1d83f2a5b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 11 May 2022 11:27:46 +0200 Subject: [PATCH 22/46] Change the TTS voice --- Sources/Navigator/TTS/AVTTSEngine.swift | 11 ++++++++++- Sources/Navigator/TTS/TTSController.swift | 6 +++--- Sources/Navigator/TTS/TTSEngine.swift | 12 +++++++----- Sources/Shared/Publication/Metadata.swift | 5 ++++- .../Services/Search/StringSearchService.swift | 2 +- Sources/Shared/Toolkit/Language.swift | 6 ++++++ TestApp/Sources/Reader/Common/TTS/TTSView.swift | 9 ++++++++- TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift | 4 ++++ 8 files changed, 43 insertions(+), 12 deletions(-) diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index dc171b96c..274bf9f12 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -73,10 +73,19 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(config.pitch)) avUtterance.preUtteranceDelay = utterance.delay avUtterance.postUtteranceDelay = config.delay - avUtterance.voice = AVSpeechSynthesisVoice(language: utterance.language ?? config.defaultLanguage) + avUtterance.voice = voice(for: utterance) return avUtterance } + private func voice(for utterance: TTSUtterance) -> AVSpeechSynthesisVoice? { + let language = utterance.language ?? config.defaultLanguage + if let voice = config.voice, voice.language == language { + return AVSpeechSynthesisVoice(identifier: voice.identifier) + } else { + return AVSpeechSynthesisVoice(language: language) + } + } + public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { guard let utterance = (utterance as? AVUtterance)?.utterance else { return diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index 5b9857232..a20de6720 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -74,8 +74,8 @@ public class TTSController: Loggable, TTSEngineDelegate { ) } - if let language = publication.metadata.languages.first { - engine.config.defaultLanguage = Language(code: .bcp47(language)) + if let language = publication.metadata.language { + engine.config.defaultLanguage = language } engine.delegate = self } @@ -260,7 +260,7 @@ public class TTSController: Loggable, TTSEngineDelegate { return TTSUtterance( text: text, locator: locator, - language: language, + language: language.takeIf { $0 != publication.metadata.language }, delay: delay ) } diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index 6ba57a8d4..e3069d305 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -1,5 +1,5 @@ // -// Copyright 2021 Readium Foundation. All rights reserved. +// Copyright 2022 Readium Foundation. All rights reserved. // Use of this source code is governed by the BSD-style license // available in the top-level LICENSE file of the project. // @@ -23,7 +23,9 @@ public protocol TTSEngineDelegate: AnyObject { } public struct TTSConfiguration { - public var defaultLanguage: Language + public var defaultLanguage: Language { + didSet { voice = nil } + } public var rate: Double public var pitch: Double public var voice: TTSVoice? @@ -44,12 +46,12 @@ public struct TTSConfiguration { } } -public struct TTSVoice { - public enum Gender { +public struct TTSVoice: Hashable { + public enum Gender: Hashable { case female, male, unspecified } - public enum Quality { + public enum Quality: Hashable { case low, medium, high } diff --git a/Sources/Shared/Publication/Metadata.swift b/Sources/Shared/Publication/Metadata.swift index 7c502f356..40d85690b 100644 --- a/Sources/Shared/Publication/Metadata.swift +++ b/Sources/Shared/Publication/Metadata.swift @@ -30,8 +30,9 @@ public struct Metadata: Hashable, Loggable, WarningLogger { public let modified: Date? public let published: Date? - // FIXME: Use `Language` instead of raw String public let languages: [String] // BCP 47 tag + // Main language of the publication. + public let language: Language? public let sortAs: String? public let subjects: [Subject] public let authors: [Contributor] @@ -104,6 +105,7 @@ public struct Metadata: Hashable, Loggable, WarningLogger { self.modified = modified self.published = published self.languages = languages + self.language = languages.first.map { Language(code: .bcp47($0)) } self.sortAs = sortAs self.subjects = subjects self.authors = authors @@ -152,6 +154,7 @@ public struct Metadata: Hashable, Loggable, WarningLogger { self.modified = parseDate(json.pop("modified")) self.published = parseDate(json.pop("published")) self.languages = parseArray(json.pop("language"), allowingSingle: true) + self.language = languages.first.map { Language(code: .bcp47($0)) } self.sortAs = json.pop("sortAs") as? String self.subjects = [Subject](json: json.pop("subject"), warnings: warnings) self.authors = [Contributor](json: json.pop("author"), warnings: warnings, normalizeHREF: normalizeHREF) diff --git a/Sources/Shared/Publication/Services/Search/StringSearchService.swift b/Sources/Shared/Publication/Services/Search/StringSearchService.swift index 7bff27584..e7e34f23f 100644 --- a/Sources/Shared/Publication/Services/Search/StringSearchService.swift +++ b/Sources/Shared/Publication/Services/Search/StringSearchService.swift @@ -25,7 +25,7 @@ public class _StringSearchService: _SearchService { return { context in _StringSearchService( publication: context.publication, - language: context.manifest.metadata.languages.first.map { Language(code: .bcp47($0)) }, + language: context.manifest.metadata.language, snippetLength: snippetLength, searchAlgorithm: searchAlgorithm, extractorFactory: extractorFactory diff --git a/Sources/Shared/Toolkit/Language.swift b/Sources/Shared/Toolkit/Language.swift index e699f036d..d031618d4 100644 --- a/Sources/Shared/Toolkit/Language.swift +++ b/Sources/Shared/Toolkit/Language.swift @@ -39,4 +39,10 @@ public struct Language: Hashable { public init(locale: Locale) { self.init(code: .bcp47(locale.identifier)) } +} + +extension Language: CustomStringConvertible { + public var description: String { + code.bcp47 + } } \ No newline at end of file diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index e1b66b4f4..40daf17b8 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -1,5 +1,5 @@ // -// Copyright 2021 Readium Foundation. All rights reserved. +// Copyright 2022 Readium Foundation. All rights reserved. // Use of this source code is governed by the BSD-style license // available in the top-level LICENSE file of the project. // @@ -89,6 +89,13 @@ struct TTSSettings: View { choices: viewModel.availableLanguages, choiceLabel: { $0.localizedName } ) + + ConfigPicker( + caption: "Voice", + for: \.voice, + choices: viewModel.availableVoices(for: viewModel.config.defaultLanguage), + choiceLabel: { $0?.name ?? "Default" } + ) } } .listStyle(.insetGrouped) diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index b51e6a8d3..2e86ab636 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -58,6 +58,10 @@ final class TTSViewModel: ObservableObject, Loggable { var defaultConfig: TTSConfiguration { tts.defaultConfig } + func availableVoices(for language: Language) -> [TTSVoice?] { + [nil] + tts.availableVoices.filter { $0.language == language } + } + lazy var availableLanguages: [Language] = tts.availableVoices .map { $0.language } From 2e5fbd1ff722cb26d6cbd12318e01e6adc6bde4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 11 May 2022 13:27:18 +0200 Subject: [PATCH 23/46] Fix setting voice and language --- Sources/Navigator/TTS/AVTTSEngine.swift | 2 +- Sources/Navigator/TTS/TTSEngine.swift | 7 +++++-- Sources/Shared/Toolkit/Language.swift | 20 +++++++++++++++++-- .../Sources/Reader/Common/TTS/TTSView.swift | 19 +++++++++++++++--- .../Reader/Common/TTS/TTSViewModel.swift | 9 +++++---- 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index 274bf9f12..a8ae7d372 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -79,7 +79,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg private func voice(for utterance: TTSUtterance) -> AVSpeechSynthesisVoice? { let language = utterance.language ?? config.defaultLanguage - if let voice = config.voice, voice.language == language { + if let voice = config.voice, voice.language.removingRegion() == language.removingRegion() { return AVSpeechSynthesisVoice(identifier: voice.identifier) } else { return AVSpeechSynthesisVoice(language: language) diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index e3069d305..cd6731a1d 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -24,7 +24,10 @@ public protocol TTSEngineDelegate: AnyObject { public struct TTSConfiguration { public var defaultLanguage: Language { - didSet { voice = nil } + didSet { + defaultLanguage = defaultLanguage.removingRegion() + voice = nil + } } public var rate: Double public var pitch: Double @@ -38,7 +41,7 @@ public struct TTSConfiguration { voice: TTSVoice? = nil, delay: TimeInterval = 0 ) { - self.defaultLanguage = defaultLanguage + self.defaultLanguage = defaultLanguage.removingRegion() self.rate = rate self.pitch = pitch self.voice = voice diff --git a/Sources/Shared/Toolkit/Language.swift b/Sources/Shared/Toolkit/Language.swift index d031618d4..6083ba5d9 100644 --- a/Sources/Shared/Toolkit/Language.swift +++ b/Sources/Shared/Toolkit/Language.swift @@ -21,17 +21,29 @@ public struct Language: Hashable { return code } } + + public func removingRegion() -> Code { + .bcp47(String(bcp47.prefix { $0 != "-" && $0 != "_" })) + } } public let code: Code public var locale: Locale { Locale(identifier: code.bcp47) } - public var localizedName: String { - Locale.current.localizedString(forIdentifier: code.bcp47) + public func localizedDescription(in locale: Locale = Locale.current) -> String { + locale.localizedString(forIdentifier: code.bcp47) ?? code.bcp47 } + public func localizedLanguage(in targetLocale: Locale = Locale.current) -> String? { + locale.languageCode.flatMap { targetLocale.localizedString(forLanguageCode: $0) } + } + + public func localizedRegion(in targetLocale: Locale = Locale.current) -> String? { + locale.regionCode.flatMap { targetLocale.localizedString(forRegionCode: $0) } + } + public init(code: Code) { self.code = code } @@ -39,6 +51,10 @@ public struct Language: Hashable { public init(locale: Locale) { self.init(code: .bcp47(locale.identifier)) } + + public func removingRegion() -> Language { + Language(code: code.removingRegion()) + } } extension Language: CustomStringConvertible { diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index 40daf17b8..e828b5b35 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -87,14 +87,14 @@ struct TTSSettings: View { caption: "Language", for: \.defaultLanguage, choices: viewModel.availableLanguages, - choiceLabel: { $0.localizedName } + choiceLabel: { $0.localizedDescription() } ) ConfigPicker( caption: "Voice", for: \.voice, - choices: viewModel.availableVoices(for: viewModel.config.defaultLanguage), - choiceLabel: { $0?.name ?? "Default" } + choices: viewModel.availableVoices, + choiceLabel: { $0.localizedDescription() } ) } } @@ -143,3 +143,16 @@ struct TTSSettings: View { } } } + +private extension Optional where Wrapped == TTSVoice { + func localizedDescription() -> String { + guard case let .some(voice) = self else { + return "Default" + } + var desc = voice.name + if let region = voice.language.localizedRegion() { + desc += " (\(region))" + } + return desc + } +} diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index 2e86ab636..5a7fe2ed3 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -58,15 +58,16 @@ final class TTSViewModel: ObservableObject, Loggable { var defaultConfig: TTSConfiguration { tts.defaultConfig } - func availableVoices(for language: Language) -> [TTSVoice?] { - [nil] + tts.availableVoices.filter { $0.language == language } + var availableVoices: [TTSVoice?] { + [nil] + tts.availableVoices + .filter { $0.language.removingRegion() == config.defaultLanguage } } lazy var availableLanguages: [Language] = tts.availableVoices - .map { $0.language } + .map { $0.language.removingRegion() } .removingDuplicates() - .sorted { $0.localizedName < $1.localizedName } + .sorted { $0.localizedDescription() < $1.localizedDescription() } @objc func play() { navigator.findLocationOfFirstVisibleContent { [self] locator in From 43580fc6306f7e8da82a2ccafe2cd9cae58dc7b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 18 May 2022 15:58:11 +0200 Subject: [PATCH 24/46] Add `TTSEngine.voiceWithIdentifier()` --- Sources/Navigator/TTS/AVTTSEngine.swift | 28 +++++++++++++++-------- Sources/Navigator/TTS/TTSController.swift | 4 ++++ Sources/Navigator/TTS/TTSEngine.swift | 1 + 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index a8ae7d372..4bfa0cc77 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -48,15 +48,13 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg } public lazy var availableVoices: [TTSVoice] = - AVSpeechSynthesisVoice.speechVoices().map { v in - TTSVoice( - identifier: v.identifier, - language: Language(code: .bcp47(v.language)), - name: v.name, - gender: .init(voice: v), - quality: .init(voice: v) - ) - } + AVSpeechSynthesisVoice.speechVoices() + .map { TTSVoice(voice: $0) } + + public func voiceWithIdentifier(_ id: String) -> TTSVoice? { + AVSpeechSynthesisVoice(identifier: id) + .map { TTSVoice(voice: $0) } + } public func speak(_ utterance: TTSUtterance) { synthesizer.stopSpeaking(at: .immediate) @@ -123,6 +121,18 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg } } +private extension TTSVoice { + init(voice: AVSpeechSynthesisVoice) { + self.init( + identifier: voice.identifier, + language: Language(code: .bcp47(voice.language)), + name: voice.name, + gender: Gender(voice: voice), + quality: Quality(voice: voice) + ) + } +} + private extension TTSVoice.Gender { init(voice: AVSpeechSynthesisVoice) { if #available(iOS 13.0, *) { diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index a20de6720..b86659888 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -49,6 +49,10 @@ public class TTSController: Loggable, TTSEngineDelegate { engine.availableVoices } + public func voiceWithIdentifier(_ id: String) -> TTSVoice? { + engine.voiceWithIdentifier(id) + } + public weak var delegate: TTSControllerDelegate? private let publication: Publication diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index cd6731a1d..f964deff1 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -12,6 +12,7 @@ public protocol TTSEngine: AnyObject { var config: TTSConfiguration { get set } var delegate: TTSEngineDelegate? { get set } var availableVoices: [TTSVoice] { get } + func voiceWithIdentifier(_ id: String) -> TTSVoice? func speak(_ utterance: TTSUtterance) func stop() From 108b2bfcff6a749336f22c842f457cc009277a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Wed, 18 May 2022 20:05:24 +0200 Subject: [PATCH 25/46] Fix flashing pages when following the speech --- Sources/Navigator/Toolkit/PaginationView.swift | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Sources/Navigator/Toolkit/PaginationView.swift b/Sources/Navigator/Toolkit/PaginationView.swift index ef181be56..1db061fb5 100644 --- a/Sources/Navigator/Toolkit/PaginationView.swift +++ b/Sources/Navigator/Toolkit/PaginationView.swift @@ -302,7 +302,16 @@ final class PaginationView: UIView, Loggable { guard 0.. Void) { func fade(to alpha: CGFloat, completion: @escaping () -> ()) { if animated { UIView.animate(withDuration: 0.15, animations: { @@ -319,10 +328,8 @@ final class PaginationView: UIView, Loggable { fade(to: 1, completion: completion) } } - - return true } - + private func scrollToView(at index: Int, location: PageLocation, completion: @escaping () -> Void) { guard currentIndex != index else { if let view = currentView { From acc3eae98c4f8a3c301328e54c840de18e29a93f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 19 May 2022 15:26:16 +0200 Subject: [PATCH 26/46] Fix deadlock with AVTTSEngine --- Sources/Navigator/TTS/AVTTSEngine.swift | 244 ++++++++++++++++-- Sources/Navigator/TTS/TTSController.swift | 4 +- Sources/Navigator/TTS/TTSEngine.swift | 2 +- .../Reader/Common/ReaderViewController.swift | 2 +- .../Reader/Common/TTS/TTSViewModel.swift | 4 +- 5 files changed, 225 insertions(+), 31 deletions(-) diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index 4bfa0cc77..1f8ed2a7b 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -8,6 +8,7 @@ import AVFoundation import Foundation import R2Shared +/// Implementation of a `TTSEngine` using Apple AVFoundation's `AVSpeechSynthesizer`. public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Loggable { /// Range of valid values for an AVUtterance rate. @@ -28,12 +29,17 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg public let defaultConfig: TTSConfiguration public var config: TTSConfiguration + private let debug: Bool public weak var delegate: TTSEngineDelegate? private let synthesizer = AVSpeechSynthesizer() - public override init() { + /// Creates a new `AVTTSEngine` instance. + /// + /// - Parameters: + /// - debug: Print the state machine transitions. + public init(debug: Bool = false) { let config = TTSConfiguration( defaultLanguage: Language(code: .bcp47(AVSpeechSynthesisVoice.currentLanguageCode())), rate: avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)), @@ -42,6 +48,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg self.defaultConfig = config self.config = config + self.debug = debug super.init() synthesizer.delegate = self @@ -57,43 +64,32 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg } public func speak(_ utterance: TTSUtterance) { - synthesizer.stopSpeaking(at: .immediate) - synthesizer.speak(avUtterance(from: utterance)) + on(.play(utterance)) } public func stop() { - synthesizer.stopSpeaking(at: .immediate) + on(.stop) } - - private func avUtterance(from utterance: TTSUtterance) -> AVSpeechUtterance { - let avUtterance = AVUtterance(utterance: utterance) - avUtterance.rate = Float(avRateRange.valueForPercentage(config.rate)) - avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(config.pitch)) - avUtterance.preUtteranceDelay = utterance.delay - avUtterance.postUtteranceDelay = config.delay - avUtterance.voice = voice(for: utterance) - return avUtterance - } - - private func voice(for utterance: TTSUtterance) -> AVSpeechSynthesisVoice? { - let language = utterance.language ?? config.defaultLanguage - if let voice = config.voice, voice.language.removingRegion() == language.removingRegion() { - return AVSpeechSynthesisVoice(identifier: voice.identifier) - } else { - return AVSpeechSynthesisVoice(language: language) + + + // MARK: AVSpeechSynthesizerDelegate + + public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didStart utterance: AVSpeechUtterance) { + guard let utterance = (utterance as? AVUtterance)?.utterance else { + return } + on(.didStart(utterance)) } public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { guard let utterance = (utterance as? AVUtterance)?.utterance else { return } - delegate?.ttsEngine(self, didFinish: utterance) + on(.didFinish(utterance)) } public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance avUtterance: AVSpeechUtterance) { guard - let delegate = delegate, let utterance = (avUtterance as? AVUtterance)?.utterance, let highlight = utterance.locator.text.highlight, let range = Range(characterRange, in: highlight) @@ -104,7 +100,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg let rangeLocator = utterance.locator.copy( text: { text in text = text[range] } ) - delegate.ttsEngine(self, willSpeakRangeAt: rangeLocator, of: utterance) + on(.willSpeakRange(locator: rangeLocator, utterance: utterance)) } private class AVUtterance: AVSpeechUtterance { @@ -119,6 +115,204 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg fatalError("Not supported") } } + + + // MARK: State machine + + // Submitting new utterances to `AVSpeechSynthesizer` when the `didStart` or + // `didFinish` events for the previous utterance were not received triggers + // a deadlock on iOS 15. The engine ignores the following requests. + // + // The following state machine is used to make sure we never send commands + // to the `AVSpeechSynthesizer` when it's not ready. + // + // To visualize it, paste the following dot graph in https://edotor.net + /* + digraph { + { + stopped [style=filled] + } + + stopped -> starting [label = "play"] + + starting -> playing [label = "didStart"] + starting -> stopping [label = "play/stop"] + + playing -> stopped [label = "didFinish"] + playing -> stopping [label = "play/stop"] + playing -> playing [label = "willSpeakRange"] + + stopping -> stopping [label = "play/stop"] + stopping -> stopping [label = "didStart"] + stopping -> starting [label = "didFinish w/ next"] + stopping -> stopped [label = "didFinish w/o next"] + } + */ + + /// Represents a state of the TTS engine. + private enum State: Equatable { + /// The TTS engine is waiting for the next utterance to play. + case stopped + /// A new utterance is being processed by the TTS engine, we wait for didStart. + case starting(TTSUtterance) + /// The utterance is currently playing and the engine is ready to process other commands. + case playing(TTSUtterance) + /// The engine was stopped while processing the previous utterance, we wait for didStart + /// and/or didFinish. The queued utterance will be played once the engine is successfully stopped. + case stopping(TTSUtterance, queued: TTSUtterance?) + + mutating func on(_ event: Event) -> Effect? { + switch (self, event) { + + // stopped + + case let (.stopped, .play(utterance)): + self = .starting(utterance) + return .play(utterance) + + // starting + + case let (.starting(current), .didStart(started)) where current == started: + self = .playing(current) + return nil + + case let (.starting(current), .play(next)): + self = .stopping(current, queued: next) + return nil + + case let (.starting(current), .stop): + self = .stopping(current, queued: nil) + return nil + + // playing + + case let (.playing(current), .didFinish(finished)) where current == finished: + self = .stopped + return .notifyDidStopAfterLastUtterance(current) + + case let (.playing(current), .play(next)): + self = .stopping(current, queued: next) + return .stop + + case let (.playing(current), .stop): + self = .stopping(current, queued: nil) + return .stop + + case let (.playing(current), .willSpeakRange(locator: Locator, utterance: speaking)) where current == speaking: + return .notifyWillSpeakRange(locator: Locator, utterance: current) + + // stopping + + case let (.stopping(current, queued: next), .didStart(started)) where current == started: + self = .stopping(current, queued: next) + return .stop + + case let (.stopping(current, queued: next), .didFinish(finished)) where current == finished: + if let next = next { + self = .starting(next) + return .play(next) + } else { + self = .stopped + return .notifyDidStopAfterLastUtterance(current) + } + + case let (.stopping(current, queued: _), .play(next)): + self = .stopping(current, queued: next) + return nil + + case let (.stopping(current, queued: _), .stop): + self = .stopping(current, queued: nil) + return nil + + + default: + return nil + } + } + } + + /// State machine events triggered by the `AVSpeechSynthesizer` or the client + /// of `AVTTSEngine`. + private enum Event: Equatable { + // AVTTSEngine commands + case play(TTSUtterance) + case stop + + // AVSpeechSynthesizer delegate events + case didStart(TTSUtterance) + case willSpeakRange(locator: Locator, utterance: TTSUtterance) + case didFinish(TTSUtterance) + } + + /// State machine side effects triggered by a state transition from an event. + private enum Effect: Equatable { + // Ask `AVSpeechSynthesizer` to play the utterance. + case play(TTSUtterance) + // Ask `AVSpeechSynthesizer` to stop the playback. + case stop + + // Send notifications to our delegate. + case notifyWillSpeakRange(locator: Locator, utterance: TTSUtterance) + case notifyDidStopAfterLastUtterance(TTSUtterance) + } + + private var state: State = .stopped { + didSet { + if (debug) { + log(.debug, "* \(state)") + } + } + } + + /// Raises a TTS event triggering a state change and handles its side effects. + private func on(_ event: Event) { + assert(Thread.isMainThread, "Raising AVTTSEngine events must be done from the main thread") + + if (debug) { + log(.debug, "-> on \(event)") + } + + if let effect = state.on(event) { + handle(effect) + } + } + + /// Handles a state machine side effect. + private func handle(_ effect: Effect) { + switch effect { + + case let .play(utterance): + synthesizer.speak(avUtterance(from: utterance)) + + case .stop: + synthesizer.stopSpeaking(at: .immediate) + + case let .notifyWillSpeakRange(locator: Locator, utterance: utterance): + delegate?.ttsEngine(self, willSpeakRangeAt: Locator, of: utterance) + + case let .notifyDidStopAfterLastUtterance(utterance): + delegate?.ttsEngine(self, didStopAfterLastUtterance: utterance) + } + } + + private func avUtterance(from utterance: TTSUtterance) -> AVSpeechUtterance { + let avUtterance = AVUtterance(utterance: utterance) + avUtterance.rate = Float(avRateRange.valueForPercentage(config.rate)) + avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(config.pitch)) + avUtterance.preUtteranceDelay = utterance.delay + avUtterance.postUtteranceDelay = config.delay + avUtterance.voice = voice(for: utterance) + return avUtterance + } + + private func voice(for utterance: TTSUtterance) -> AVSpeechSynthesisVoice? { + let language = utterance.language ?? config.defaultLanguage + if let voice = config.voice, voice.language.removingRegion() == language.removingRegion() { + return AVSpeechSynthesisVoice(identifier: voice.identifier) + } else { + return AVSpeechSynthesisVoice(language: language) + } + } } private extension TTSVoice { @@ -169,4 +363,4 @@ private extension AVSpeechSynthesisVoice { convenience init?(language: Language) { self.init(language: language.code.bcp47) } -} \ No newline at end of file +} diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index b86659888..38c037e10 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -280,8 +280,8 @@ public class TTSController: Loggable, TTSEngineDelegate { // MARK: - TTSEngineDelegate - public func ttsEngine(_ engine: TTSEngine, didFinish utterance: TTSUtterance) { - if isPlaying && currentUtterance == utterance { + public func ttsEngine(_ engine: TTSEngine, didStopAfterLastUtterance utterance: TTSUtterance) { + if isPlaying { next() } } diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index f964deff1..03975a145 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -20,7 +20,7 @@ public protocol TTSEngine: AnyObject { public protocol TTSEngineDelegate: AnyObject { func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) - func ttsEngine(_ engine: TTSEngine, didFinish utterance: TTSUtterance) + func ttsEngine(_ engine: TTSEngine, didStopAfterLastUtterance utterance: TTSUtterance) } public struct TTSConfiguration { diff --git a/TestApp/Sources/Reader/Common/ReaderViewController.swift b/TestApp/Sources/Reader/Common/ReaderViewController.swift index 02d1b8160..851456d4f 100644 --- a/TestApp/Sources/Reader/Common/ReaderViewController.swift +++ b/TestApp/Sources/Reader/Common/ReaderViewController.swift @@ -124,7 +124,7 @@ class ReaderViewController: UIViewController, Loggable { controls.didMove(toParent: self) state - .sink { [unowned self] state in + .sink { state in controls.view.isHidden = (state == .stopped) } .store(in: &subscriptions) diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index 5a7fe2ed3..c84d9f2e7 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -45,7 +45,7 @@ final class TTSViewModel: ObservableObject, Loggable { var isMoving = false playingRangeLocatorSubject .throttle(for: 1, scheduler: RunLoop.main, latest: true) - .sink { [unowned self] locator in + .sink { locator in guard !isMoving else { return } @@ -131,4 +131,4 @@ extension TTSViewModel: TTSControllerDelegate { public func ttsController(_ ttsController: TTSController, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) { playingRangeLocatorSubject.send(locator) } -} \ No newline at end of file +} From e682b7bb8f1f140dace0aa87d11a5e2a11a47997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 19 May 2022 16:46:47 +0200 Subject: [PATCH 27/46] Setup last position when jumping to a previous resource --- .../Content/HTMLResourceContentIterator.swift | 43 +++++++++++++------ .../Content/PublicationContentIterator.swift | 29 +++++++------ 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift b/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift index 3b104a8e4..96744722b 100644 --- a/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift @@ -25,22 +25,12 @@ public class HTMLResourceContentIterator : ContentIterator { .readAsString() .eraseToAnyError() .tryMap { try SwiftSoup.parse($0) } - .tryMap { document -> ContentParser in - let parser = ContentParser( - baseLocator: locator, - startElement: try locator.locations.cssSelector - .flatMap { - // The JS third-party library used to generate the CSS Selector sometimes adds - // :root >, which doesn't work with JSoup. - try document.select($0.removingPrefix(":root > ")).first() - } - ) - try document.traverse(parser) - return parser + .tryMap { document -> (content: [Content], startingIndex: Int) in + try ContentParser.parse(document: document, locator: locator) } content = result.map { $0.content } - startingIndex = result.map { $0.startIndex }.get(or: 0) + startingIndex = result.map { $0.startingIndex }.get(or: 0) } public func close() {} @@ -72,6 +62,31 @@ public class HTMLResourceContentIterator : ContentIterator { } private class ContentParser: NodeVisitor { + + static func parse(document: Document, locator: Locator) throws -> (content: [Content], startingIndex: Int) { + let parser = ContentParser( + baseLocator: locator, + startElement: try locator.locations.cssSelector + .flatMap { + // The JS third-party library used to generate the CSS Selector sometimes adds + // :root >, which doesn't work with JSoup. + try document.select($0.removingPrefix(":root > ")).first() + } + ) + try document.traverse(parser) + + var result = ( + content: parser.content, + startingIndex: parser.startIndex + ) + + if locator.locations.progression == 1.0 { + result.startingIndex = result.content.count - 1 + } + + return result + } + private let baseLocator: Locator private let startElement: Element? @@ -87,7 +102,7 @@ public class HTMLResourceContentIterator : ContentIterator { private var currentCSSSelector: String? private var ignoredNode: Node? - init(baseLocator: Locator, startElement: Element?) { + private init(baseLocator: Locator, startElement: Element?) { self.baseLocator = baseLocator self.startElement = startElement } diff --git a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift index 8e5aed100..c3c1ac164 100644 --- a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift @@ -87,28 +87,31 @@ public class PublicationContentIterator: ContentIterator, Loggable { private func loadIterator(from index: Int, by delta: Int) -> (index: Int, ContentIterator)? { let i = index + delta - guard publication.readingOrder.indices.contains(i) else { + guard + let link = publication.readingOrder.getOrNil(i), + var locator = publication.locate(link) + else { return nil } - guard let iterator = loadIterator(at: i) else { - return loadIterator(from: i, by: delta) - } - return (i, iterator) - } - - private func loadIterator(at index: Int) -> ContentIterator? { - let link = publication.readingOrder[index] - guard var locator = publication.locate(link) else { - return nil - } - + if let start = startLocator.pop() { locator = locator.copy( locations: { $0 = start.locations }, text: { $0 = start.text } ) + } else if delta < 0 { + locator = locator.copy( + locations: { $0.progression = 1.0 } + ) + } + + guard let iterator = loadIterator(at: link, locator: locator) else { + return loadIterator(from: i, by: delta) } + return (i, iterator) + } + private func loadIterator(at link: Link, locator: Locator) -> ContentIterator? { let resource = publication.get(link) for factory in resourceContentIteratorFactories { if let iterator = factory(resource, locator) { From c9642fb37036ea26c838cda1c5d38f4f0a986b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Thu, 19 May 2022 17:54:55 +0200 Subject: [PATCH 28/46] Improve settings screen --- .../Sources/Reader/Common/TTS/TTSView.swift | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index e828b5b35..69790bf78 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -51,7 +51,7 @@ struct TTSControls: View { TTSSettings(viewModel: viewModel) .frame( minWidth: 320, idealWidth: 400, maxWidth: nil, - minHeight: 150, idealHeight: 150, maxHeight: nil, + minHeight: 300, idealHeight: 300, maxHeight: nil, alignment: .top ) } @@ -69,48 +69,47 @@ struct TTSSettings: View { @ObservedObject var viewModel: TTSViewModel var body: some View { - List { - Section(header: Text("Speech settings")) { - ConfigStepper( + NavigationView { + Form { + stepper( caption: "Rate", for: \.rate, step: viewModel.defaultConfig.rate / 10 ) - ConfigStepper( + stepper( caption: "Pitch", for: \.pitch, step: viewModel.defaultConfig.pitch / 4 ) - ConfigPicker( + picker( caption: "Language", for: \.defaultLanguage, choices: viewModel.availableLanguages, choiceLabel: { $0.localizedDescription() } ) - ConfigPicker( + picker( caption: "Voice", for: \.voice, choices: viewModel.availableVoices, choiceLabel: { $0.localizedDescription() } ) } + .navigationTitle("Speech settings") + .navigationBarTitleDisplayMode(.inline) } - .listStyle(.insetGrouped) + .navigationViewStyle(.stack) } - @ViewBuilder func ConfigStepper( + @ViewBuilder func stepper( caption: String, for keyPath: WritableKeyPath, step: Double ) -> some View { Stepper( - value: Binding( - get: { viewModel.config[keyPath: keyPath] }, - set: { viewModel.config[keyPath: keyPath] = $0 } - ), + value: configBinding(for: keyPath), in: 0.0...1.0, step: step ) { @@ -119,29 +118,25 @@ struct TTSSettings: View { } } - @ViewBuilder func ConfigPicker( + @ViewBuilder func picker( caption: String, for keyPath: WritableKeyPath, choices: [T], choiceLabel: @escaping (T) -> String ) -> some View { - HStack { - Text(caption) - Spacer() - - Picker(caption, - selection: Binding( - get: { viewModel.config[keyPath: keyPath] }, - set: { viewModel.config[keyPath: keyPath] = $0 } - ) - ) { - ForEach(choices, id: \.self) { - Text(choiceLabel($0)) - } + Picker(caption, selection: configBinding(for: keyPath)) { + ForEach(choices, id: \.self) { + Text(choiceLabel($0)) } - .pickerStyle(.menu) } } + + private func configBinding(for keyPath: WritableKeyPath) -> Binding { + Binding( + get: { viewModel.config[keyPath: keyPath] }, + set: { viewModel.config[keyPath: keyPath] = $0 } + ) + } } private extension Optional where Wrapped == TTSVoice { From f074a933a4dd81ae0be0b22373fa80af7af4d3af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 23 May 2022 16:15:39 +0200 Subject: [PATCH 29/46] Support for AudioSession to play while in the background --- .../Navigator/Audiobook/AudioNavigator.swift | 21 +++++++++-- Sources/Navigator/TTS/AVTTSEngine.swift | 36 +++++++++++++++++-- .../Shared/Toolkit/Media/AudioSession.swift | 17 ++++----- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/Sources/Navigator/Audiobook/AudioNavigator.swift b/Sources/Navigator/Audiobook/AudioNavigator.swift index af256f547..379e9c9a8 100644 --- a/Sources/Navigator/Audiobook/AudioNavigator.swift +++ b/Sources/Navigator/Audiobook/AudioNavigator.swift @@ -23,11 +23,28 @@ open class _AudioNavigator: _MediaNavigator, _AudioSessionUser, Loggable { private let publication: Publication private let initialLocation: Locator? - - public init(publication: Publication, initialLocation: Locator? = nil) { + public let audioConfiguration: _AudioSession.Configuration + + public init( + publication: Publication, + initialLocation: Locator? = nil, + audioConfig: _AudioSession.Configuration = .init( + category: .playback, + mode: .default, + routeSharingPolicy: { + if #available(iOS 11.0, *) { + return .longForm + } else { + return .default + } + }(), + options: [] + ) + ) { self.publication = publication self.initialLocation = initialLocation ?? publication.readingOrder.first.flatMap { publication.locate($0) } + self.audioConfiguration = audioConfig let durations = publication.readingOrder.map { $0.duration ?? 0 } self.durations = durations diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index 1f8ed2a7b..01eae0a03 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -38,8 +38,17 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// Creates a new `AVTTSEngine` instance. /// /// - Parameters: + /// - audioSessionConfig: AudioSession configuration used while playing utterances. If `nil`, utterances won't + /// play when the app is in the background. /// - debug: Print the state machine transitions. - public init(debug: Bool = false) { + public init( + audioSessionConfig: _AudioSession.Configuration? = .init( + category: .playback, + mode: .spokenAudio, + options: .mixWithOthers + ), + debug: Bool = false + ) { let config = TTSConfiguration( defaultLanguage: Language(code: .bcp47(AVSpeechSynthesisVoice.currentLanguageCode())), rate: avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)), @@ -49,6 +58,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg self.defaultConfig = config self.config = config self.debug = debug + self.audioSessionUser = audioSessionConfig.map { AudioSessionUser(config: $0) } super.init() synthesizer.delegate = self @@ -283,7 +293,11 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg case let .play(utterance): synthesizer.speak(avUtterance(from: utterance)) - + + if let user = audioSessionUser { + _AudioSession.shared.start(with: user) + } + case .stop: synthesizer.stopSpeaking(at: .immediate) @@ -313,6 +327,24 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg return AVSpeechSynthesisVoice(language: language) } } + + // MARK: - Audio session + + private let audioSessionUser: AudioSessionUser? + + private final class AudioSessionUser: R2Shared._AudioSessionUser { + let audioConfiguration: _AudioSession.Configuration + + init(config: _AudioSession.Configuration) { + self.audioConfiguration = config + } + + deinit { + _AudioSession.shared.end(for: self) + } + + func play() {} + } } private extension TTSVoice { diff --git a/Sources/Shared/Toolkit/Media/AudioSession.swift b/Sources/Shared/Toolkit/Media/AudioSession.swift index 7eec678e7..efbf57c29 100644 --- a/Sources/Shared/Toolkit/Media/AudioSession.swift +++ b/Sources/Shared/Toolkit/Media/AudioSession.swift @@ -13,7 +13,6 @@ import AVFoundation import Foundation /// An user of the `AudioSession`, for example a media player object. -@available(iOS 10.0, *) public protocol _AudioSessionUser: AnyObject { /// Audio session configuration to use for this user. @@ -25,28 +24,31 @@ public protocol _AudioSessionUser: AnyObject { } -@available(iOS 10.0, *) public extension _AudioSessionUser { - var audioConfiguration: _AudioSession.Configuration { .init() } - } /// Manages an activated `AVAudioSession`. /// /// **WARNING:** This API is experimental and may change or be removed in a future release without /// notice. Use with caution. -@available(iOS 10.0, *) public final class _AudioSession: Loggable { public struct Configuration { let category: AVAudioSession.Category let mode: AVAudioSession.Mode + let routeSharingPolicy: AVAudioSession.RouteSharingPolicy let options: AVAudioSession.CategoryOptions - public init(category: AVAudioSession.Category = .playback, mode: AVAudioSession.Mode = .default, options: AVAudioSession.CategoryOptions = []) { + public init( + category: AVAudioSession.Category = .playback, + mode: AVAudioSession.Mode = .default, + routeSharingPolicy: AVAudioSession.RouteSharingPolicy = .default, + options: AVAudioSession.CategoryOptions = [] + ) { self.category = category self.mode = mode + self.routeSharingPolicy = routeSharingPolicy self.options = options } } @@ -74,7 +76,7 @@ public final class _AudioSession: Loggable { do { let config = user.audioConfiguration if #available(iOS 11.0, *) { - try audioSession.setCategory(config.category, mode: config.mode, policy: .longForm, options: config.options) + try audioSession.setCategory(config.category, mode: config.mode, policy: config.routeSharingPolicy, options: config.options) } else { try audioSession.setCategory(config.category, mode: config.mode, options: config.options) } @@ -154,5 +156,4 @@ public final class _AudioSession: Loggable { break } } - } From 7b609f3ae220a6a7c34d5111f9e157484158a1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Fri, 22 Jul 2022 12:35:56 +0200 Subject: [PATCH 30/46] Fix Xcode warnings --- Sources/Navigator/TTS/TTSController.swift | 4 ++-- .../Reader/Common/Highlight/HighlightContextMenu.swift | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index 38c037e10..a9c7fa38f 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -157,7 +157,7 @@ public class TTSController: Loggable, TTSEngineDelegate { queue.async { [self] in do { let utterance = try nextUtterance(direction: direction) - DispatchQueue.main.async { + DispatchQueue.main.async { [self] in if let utterance = utterance { play(utterance) } else { @@ -165,7 +165,7 @@ public class TTSController: Loggable, TTSEngineDelegate { } } } catch { - DispatchQueue.main.async { + DispatchQueue.main.async { [self] in delegate?.ttsController(self, didReceiveError: error) } } diff --git a/TestApp/Sources/Reader/Common/Highlight/HighlightContextMenu.swift b/TestApp/Sources/Reader/Common/Highlight/HighlightContextMenu.swift index 091348d8b..0cee14bb8 100644 --- a/TestApp/Sources/Reader/Common/Highlight/HighlightContextMenu.swift +++ b/TestApp/Sources/Reader/Common/Highlight/HighlightContextMenu.swift @@ -24,11 +24,11 @@ struct HighlightContextMenu: View { var body: some View { HStack { - ForEach(0.. Date: Sat, 6 Aug 2022 17:08:53 +0200 Subject: [PATCH 31/46] Refactor content models --- Sources/Navigator/TTS/TTSController.swift | 8 +- .../Services/Content/Content.swift | 217 ++++++++++++++++++ .../Content/ContentIterationService.swift | 6 +- ...terator.swift => ContentIteratorOld.swift} | 8 +- .../Services/Content/ContentService.swift | 86 +++++++ .../Content/HTMLResourceContentIterator.swift | 24 +- .../Content/PublicationContentIterator.swift | 16 +- .../Toolkit/Tokenizer/ContentTokenizer.swift | 10 +- 8 files changed, 339 insertions(+), 36 deletions(-) create mode 100644 Sources/Shared/Publication/Services/Content/Content.swift rename Sources/Shared/Publication/Services/Content/{ContentIterator.swift => ContentIteratorOld.swift} (89%) create mode 100644 Sources/Shared/Publication/Services/Content/ContentService.swift diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index a9c7fa38f..ef9d548ae 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -142,7 +142,7 @@ public class TTSController: Loggable, TTSEngineDelegate { case forward, backward } - private var contentIterator: ContentIterator? { + private var contentIterator: ContentIteratorOld? { willSet { contentIterator?.close() } } @@ -211,7 +211,7 @@ public class TTSController: Loggable, TTSEngineDelegate { speakingUtteranceIndex = nil utterances = [] - guard let content: Content = try { + guard let content: ContentOld = try { switch direction { case .forward: return try contentIterator?.next() @@ -232,7 +232,7 @@ public class TTSController: Loggable, TTSEngineDelegate { return true } - private func utterances(from content: Content) -> [TTSUtterance] { + private func utterances(from content: ContentOld) -> [TTSUtterance] { switch content.data { case .audio(target: _): return [] @@ -269,7 +269,7 @@ public class TTSController: Loggable, TTSEngineDelegate { ) } - private func tokenize(_ content: Content, with tokenizer: ContentTokenizer) -> [Content] { + private func tokenize(_ content: ContentOld, with tokenizer: ContentTokenizer) -> [ContentOld] { do { return try tokenizer(content) } catch { diff --git a/Sources/Shared/Publication/Services/Content/Content.swift b/Sources/Shared/Publication/Services/Content/Content.swift new file mode 100644 index 000000000..5618a25c6 --- /dev/null +++ b/Sources/Shared/Publication/Services/Content/Content.swift @@ -0,0 +1,217 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// Provides an iterable list of `ContentElement`s. +public protocol Content { + func makeIterator() -> ContentIterator +} + +/// Represents a single semantic content element part of a publication. +public protocol ContentElement: ContentAttributesHolder { + /// Locator targeting this element in the Publication. + var locator: Locator { get } +} + +/// An element which can be represented as human-readable text. +/// +/// The default implementation returns the first accessibility label associated to the element. +public protocol TextualContentElement: ContentElement { + /// Human-readable text representation for this element. + var text: String? { get } +} + +public extension TextualContentElement { + var text: String? { accessibilityLabel } +} + +/// An element referencing an embedded external resource. +public protocol EmbeddedContentElement: ContentElement { + /// Referenced resource in the publication. + var embeddedLink: Link { get } +} + +/// An audio clip. +public struct AudioContentElement: EmbeddedContentElement, TextualContentElement { + public let locator: Locator + public let embeddedLink: Link + public let attributes: [ContentAttribute] + + public init(locator: Locator, embeddedLink: Link, attributes: [ContentAttribute] = []) { + self.locator = locator + self.embeddedLink = embeddedLink + self.attributes = attributes + } +} + +/// A video clip. +public struct VideoContentElement: EmbeddedContentElement, TextualContentElement { + public let locator: Locator + public let embeddedLink: Link + public let attributes: [ContentAttribute] + + public init(locator: Locator, embeddedLink: Link, attributes: [ContentAttribute] = []) { + self.locator = locator + self.embeddedLink = embeddedLink + self.attributes = attributes + } +} + +/// A bitmap image. +public struct ImageContentElement: EmbeddedContentElement, TextualContentElement { + public let locator: Locator + public let embeddedLink: Link + + /// Short piece of text associated with the image. + public let caption: String? + public let attributes: [ContentAttribute] + + public init(locator: Locator, embeddedLink: Link, caption: String? = nil, attributes: [ContentAttribute] = []) { + self.locator = locator + self.embeddedLink = embeddedLink + self.caption = caption + self.attributes = attributes + } + + public var text: String? { + // The caption might be a better text description than the accessibility label, when available. + caption.takeIf { !$0.isEmpty } ?? (self as? TextualContentElement)?.text + } +} + +/// A text element. +/// +/// @param role Purpose of this element in the broader context of the document. +/// @param segments Ranged portions of text with associated attributes. +public struct TextContentElement: TextualContentElement { + public let locator: Locator + public let role: Role + public let segments: [Segment] + public let attributes: [ContentAttribute] + + public init(locator: Locator, role: Role, segments: [Segment], attributes: [ContentAttribute] = []) { + self.locator = locator + self.role = role + self.segments = segments + self.attributes = attributes + } + + public var text: String? { + segments.map(\.text).joined() + } + + + /// Represents a purpose of an element in the broader context of the document. + public enum Role { + /// Title of a section with its level (1 being the highest). + case heading(level: Int) + + /// Normal body of content. + case body + + /// A footnote at the bottom of a document. + case footnote + + /// A quotation. + case quote(referenceUrl: URL?, referenceTitle: String?) + } + + /// Ranged portion of text with associated attributes. + /// + /// @param locator Locator to the segment of text. + /// @param text Text in the segment. + /// @param attributes Attributes associated with this segment, e.g. language. + public struct Segment { + public let locator: Locator + public let text: String + public let attributes: [ContentAttribute] + + public init(locator: Locator, text: String, attributes: [ContentAttribute] = []) { + self.locator = locator + self.text = text + self.attributes = attributes + } + } +} + +/// An attribute key identifies uniquely a type of attribute. +/// +/// The `V` phantom type is there to perform static type checking when requesting an attribute. +public struct ContentAttributeKey { + public static var accessibilityLabel: ContentAttributeKey { .init("accessibilityLabel") } + public static var language: ContentAttributeKey { .init("language") } + + public let key: String + public init(_ key: String) { + self.key = key + } +} + +public struct ContentAttribute: Hashable { + public let key: String + public let value: AnyHashable + + public init(key: String, value: AnyHashable) { + self.key = key + self.value = value + } +} + +/// Object associated with a list of attributes. +public protocol ContentAttributesHolder { + /// Associated list of attributes. + var attributes: [ContentAttribute] { get } +} + +public extension ContentAttributesHolder { + + var language: Language? { self[.language] } + var accessibilityLabel: String? { self[.accessibilityLabel] } + + /// Gets the first attribute with the given `key`. + subscript(_ key: ContentAttributeKey) -> T? { + attribute(key) + } + + /// Gets the first attribute with the given `key`. + func attribute(_ key: ContentAttributeKey) -> T? { + attributes.first { attr in + if attr.key == key.key, let value = attr.value as? T { + return value + } else { + return nil + } + } + } + + /// Gets all the attributes with the given `key`. + func attributes(_ key: ContentAttributeKey) -> [T] { + attributes.compactMap { attr in + if attr.key == key.key, let value = attr.value as? T { + return value + } else { + return nil + } + } + } +} + +/// Iterates through a list of `ContentElement` items. +public protocol ContentIterator: AnyObject { + + /// Returns true if the iterator has a next element, potentially blocking the caller while processing it. + func hasNext() -> Bool + + /// Retrieves the next element, or nil if we reached the end. + func next() throws -> ContentElement? + + /// Returns true if the iterator has a previous element, potentially blocking the caller while processing it. + func hasPrevious() -> Bool + + /// Advances to the previous item and returns it, or null if we reached the beginning. + func previous() throws -> ContentElement? +} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/ContentIterationService.swift b/Sources/Shared/Publication/Services/Content/ContentIterationService.swift index 0d5c441fb..53a929f0b 100644 --- a/Sources/Shared/Publication/Services/Content/ContentIterationService.swift +++ b/Sources/Shared/Publication/Services/Content/ContentIterationService.swift @@ -9,7 +9,7 @@ import Foundation public typealias ContentIterationServiceFactory = (PublicationServiceContext) -> ContentIterationService? public protocol ContentIterationService: PublicationService { - func iterator(from start: Locator?) -> ContentIterator? + func iterator(from start: Locator?) -> ContentIteratorOld? } public extension Publication { @@ -17,7 +17,7 @@ public extension Publication { contentIterationService != nil } - func contentIterator(from start: Locator?) -> ContentIterator? { + func contentIterator(from start: Locator?) -> ContentIteratorOld? { contentIterationService?.iterator(from: start) } @@ -58,7 +58,7 @@ public class DefaultContentIterationService: ContentIterationService { self.resourceContentIteratorFactories = resourceContentIteratorFactories } - public func iterator(from start: Locator?) -> ContentIterator? { + public func iterator(from start: Locator?) -> ContentIteratorOld? { guard let publication = publication() else { return nil } diff --git a/Sources/Shared/Publication/Services/Content/ContentIterator.swift b/Sources/Shared/Publication/Services/Content/ContentIteratorOld.swift similarity index 89% rename from Sources/Shared/Publication/Services/Content/ContentIterator.swift rename to Sources/Shared/Publication/Services/Content/ContentIteratorOld.swift index 320a4fec2..7a3df4526 100644 --- a/Sources/Shared/Publication/Services/Content/ContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/ContentIteratorOld.swift @@ -6,7 +6,7 @@ import Foundation -public struct Content: Equatable { +public struct ContentOld: Equatable { public let locator: Locator public let data: Data @@ -52,8 +52,8 @@ public struct Content: Equatable { } } -public protocol ContentIterator: AnyObject { +public protocol ContentIteratorOld: AnyObject { func close() - func previous() throws -> Content? - func next() throws -> Content? + func previous() throws -> ContentOld? + func next() throws -> ContentOld? } \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/ContentService.swift b/Sources/Shared/Publication/Services/Content/ContentService.swift new file mode 100644 index 000000000..d5b8cdaa0 --- /dev/null +++ b/Sources/Shared/Publication/Services/Content/ContentService.swift @@ -0,0 +1,86 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +public typealias ContentServiceFactory = (PublicationServiceContext) -> ContentService? + +/// Provides a way to extract the raw `Content` of a `Publication`. +public protocol ContentService: PublicationService { + /// Creates a `Content` starting from the given `start` location. + /// + /// The implementation must be fast and non-blocking. Do the actual extraction inside the + /// `Content` implementation. + func content(from start: Locator?) -> Content? +} + +/// Default implementation of `ContentService`, delegating the content parsing to `ResourceContentIteratorFactory`. +public class DefaultContentService: ContentService { + private let publication: Weak + private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] + + public init(publication: Weak, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { + self.publication = publication + self.resourceContentIteratorFactories = resourceContentIteratorFactories + } + + public static func makeFactory(resourceContentIteratorFactories: [ResourceContentIteratorFactory]) -> (PublicationServiceContext) -> DefaultContentService? { + { context in + DefaultContentService(publication: context.publication, resourceContentIteratorFactories: resourceContentIteratorFactories) + } + } + + public func content(from start: Locator?) -> Content? { + guard let pub = publication() else { + return nil + } + return DefaultContent(publication: pub, start: start) + } + + private class DefaultContent: Content { + let publication: Publication + let start: Locator? + + init(publication: Publication, start: Locator?) { + self.publication = publication + self.start = start + } + + func makeIterator() -> ContentIterator { + PublicationContentIterator( + publication: publication, + start: start, + resourceContentIteratorFactories: resourceContentIteratorFactories + ) + } + } +} + + +// MARK: Publication Helpers + +public extension Publication { + + /// Creates a [Content] starting from the given `start` location, or the beginning of the + /// publication when missing. + func content(from start: Locator? = nil) -> Content? { + findService(ContentService.self)?.content(from: start) + } +} + + +// MARK: PublicationServicesBuilder Helpers + +public extension PublicationServicesBuilder { + + mutating func setContentServiceFactory(_ factory: ContentServiceFactory?) { + if let factory = factory { + set(ContentService.self, factory) + } else { + remove(ContentService.self) + } + } +} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift b/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift index 96744722b..f1787f6ff 100644 --- a/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/HTMLResourceContentIterator.swift @@ -7,7 +7,7 @@ import Foundation import SwiftSoup -public class HTMLResourceContentIterator : ContentIterator { +public class HTMLResourceContentIterator : ContentIteratorOld { // FIXME: Custom skipped elements public static func makeFactory() -> ResourceContentIteratorFactory { @@ -16,7 +16,7 @@ public class HTMLResourceContentIterator : ContentIterator { } } - private var content: Result<[Content], Error> + private var content: Result<[ContentOld], Error> private var currentIndex: Int? private var startingIndex: Int @@ -25,7 +25,7 @@ public class HTMLResourceContentIterator : ContentIterator { .readAsString() .eraseToAnyError() .tryMap { try SwiftSoup.parse($0) } - .tryMap { document -> (content: [Content], startingIndex: Int) in + .tryMap { document -> (content: [ContentOld], startingIndex: Int) in try ContentParser.parse(document: document, locator: locator) } @@ -35,15 +35,15 @@ public class HTMLResourceContentIterator : ContentIterator { public func close() {} - public func previous() throws -> Content? { + public func previous() throws -> ContentOld? { try next(by: -1) } - public func next() throws -> Content? { + public func next() throws -> ContentOld? { try next(by: +1) } - private func next(by delta: Int) throws -> Content? { + private func next(by delta: Int) throws -> ContentOld? { let content = try content.get() let index = index(by: delta) guard content.indices.contains(index) else { @@ -63,7 +63,7 @@ public class HTMLResourceContentIterator : ContentIterator { private class ContentParser: NodeVisitor { - static func parse(document: Document, locator: Locator) throws -> (content: [Content], startingIndex: Int) { + static func parse(document: Document, locator: Locator) throws -> (content: [ContentOld], startingIndex: Int) { let parser = ContentParser( baseLocator: locator, startElement: try locator.locations.cssSelector @@ -90,10 +90,10 @@ public class HTMLResourceContentIterator : ContentIterator { private let baseLocator: Locator private let startElement: Element? - private(set) var content: [Content] = [] + private(set) var content: [ContentOld] = [] private(set) var startIndex = 0 private var currentElement: Element? - private var spansAcc: [Content.TextSpan] = [] + private var spansAcc: [ContentOld.TextSpan] = [] private var textAcc = StringBuilder() private var wholeRawTextAcc: String = "" private var elementRawTextAcc: String = "" @@ -130,7 +130,7 @@ public class HTMLResourceContentIterator : ContentIterator { if let href = try elem.attr("src") .takeUnlessEmpty() .map({ HREF($0, relativeTo: baseLocator.href).string }) { - content.append(Content( + content.append(ContentOld( locator: baseLocator.copy( locations: { $0 = Locator.Locations( @@ -194,7 +194,7 @@ public class HTMLResourceContentIterator : ContentIterator { if startElement != nil && currentElement == startElement { startIndex = content.count } - content.append(Content( + content.append(ContentOld( locator: baseLocator.copy( locations: { [self] in $0 = Locator.Locations( @@ -229,7 +229,7 @@ public class HTMLResourceContentIterator : ContentIterator { text = trimmedText + whitespaceSuffix } - spansAcc.append(Content.TextSpan( + spansAcc.append(ContentOld.TextSpan( locator: baseLocator.copy( locations: { [self] in $0 = Locator.Locations( diff --git a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift index c3c1ac164..80c5c2c28 100644 --- a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift @@ -10,16 +10,16 @@ import Foundation /// /// - Returns: nil if the resource format is not supported. public typealias ResourceContentIteratorFactory = - (_ resource: Resource, _ locator: Locator) -> ContentIterator? + (_ resource: Resource, _ locator: Locator) -> ContentIteratorOld? -public class PublicationContentIterator: ContentIterator, Loggable { +public class PublicationContentIterator: ContentIteratorOld, Loggable { private let publication: Publication private var startLocator: Locator? private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] private var startIndex: Int? private var currentIndex: Int = 0 - private var currentIterator: ContentIterator? + private var currentIterator: ContentIteratorOld? public init(publication: Publication, start: Locator?, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { self.publication = publication @@ -42,7 +42,7 @@ public class PublicationContentIterator: ContentIterator, Loggable { currentIterator = nil } - public func previous() throws -> Content? { + public func previous() throws -> ContentOld? { guard let iterator = iterator(by: -1) else { return nil } @@ -53,7 +53,7 @@ public class PublicationContentIterator: ContentIterator, Loggable { return content } - public func next() throws -> Content? { + public func next() throws -> ContentOld? { guard let iterator = iterator(by: +1) else { return nil } @@ -64,7 +64,7 @@ public class PublicationContentIterator: ContentIterator, Loggable { return content } - private func iterator(by delta: Int) -> ContentIterator? { + private func iterator(by delta: Int) -> ContentIteratorOld? { if let iter = currentIterator { return iter } @@ -85,7 +85,7 @@ public class PublicationContentIterator: ContentIterator, Loggable { return newIterator } - private func loadIterator(from index: Int, by delta: Int) -> (index: Int, ContentIterator)? { + private func loadIterator(from index: Int, by delta: Int) -> (index: Int, ContentIteratorOld)? { let i = index + delta guard let link = publication.readingOrder.getOrNil(i), @@ -111,7 +111,7 @@ public class PublicationContentIterator: ContentIterator, Loggable { return (i, iterator) } - private func loadIterator(at link: Link, locator: Locator) -> ContentIterator? { + private func loadIterator(at link: Link, locator: Locator) -> ContentIteratorOld? { let resource = publication.get(link) for factory in resourceContentIteratorFactories { if let iterator = factory(resource, locator) { diff --git a/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift index c1abc94a1..c9a776ccb 100644 --- a/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift +++ b/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift @@ -7,7 +7,7 @@ import Foundation /// A tokenizer splitting a `Content` into smaller pieces. -public typealias ContentTokenizer = Tokenizer +public typealias ContentTokenizer = Tokenizer /// A `ContentTokenizer` using the default `TextTokenizer` to split the text of the `Content` by `unit`. public func makeTextContentTokenizer(unit: TextUnit, language: Language?) -> ContentTokenizer { @@ -16,10 +16,10 @@ public func makeTextContentTokenizer(unit: TextUnit, language: Language?) -> Con /// A `ContentTokenizer` using a `TextTokenizer` to split the text of the `Content`. public func makeTextContentTokenizer(with tokenizer: @escaping TextTokenizer) -> ContentTokenizer { - func tokenize(_ span: Content.TextSpan) throws -> [Content.TextSpan] { + func tokenize(_ span: ContentOld.TextSpan) throws -> [ContentOld.TextSpan] { try tokenizer(span.text) .map { range in - Content.TextSpan( + ContentOld.TextSpan( locator: span.locator.copy(text: { $0 = extractTextContext(in: span.text, for: range) }), language: span.language, text: String(span.text[range]) @@ -27,12 +27,12 @@ public func makeTextContentTokenizer(with tokenizer: @escaping TextTokenizer) -> } } - func tokenize(_ content: Content) throws -> [Content] { + func tokenize(_ content: ContentOld) throws -> [ContentOld] { switch content.data { case .audio, .image: return [content] case .text(spans: let spans, style: let style): - return [Content( + return [ContentOld( locator: content.locator, data: .text( spans: try spans.flatMap { try tokenize($0) }, From 4d31aefdb1aad06c6d110ba63c69b852142ee84f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sun, 7 Aug 2022 13:56:16 +0200 Subject: [PATCH 32/46] Refactor content iterators --- .../Services/Content/Content.swift | 47 +++-- .../Content/ContentIterationService.swift | 71 -------- .../Services/Content/ContentIteratorOld.swift | 59 ------- .../Services/Content/ContentService.swift | 6 +- .../Services/Content/ContentTokenizer.swift | 58 +++++++ .../HTMLResourceContentIterator.swift | 163 ++++++++++-------- .../PublicationContentIterator.swift | 151 ++++++++++++++++ .../Content/PublicationContentIterator.swift | 124 ------------- .../Services/PublicationServicesBuilder.swift | 4 +- .../Toolkit/Tokenizer/ContentTokenizer.swift | 56 ------ Sources/Streamer/Parser/EPUB/EPUBParser.swift | 2 +- 11 files changed, 333 insertions(+), 408 deletions(-) delete mode 100644 Sources/Shared/Publication/Services/Content/ContentIterationService.swift delete mode 100644 Sources/Shared/Publication/Services/Content/ContentIteratorOld.swift create mode 100644 Sources/Shared/Publication/Services/Content/ContentTokenizer.swift rename Sources/Shared/Publication/Services/Content/{ => Iterators}/HTMLResourceContentIterator.swift (63%) create mode 100644 Sources/Shared/Publication/Services/Content/Iterators/PublicationContentIterator.swift delete mode 100644 Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift delete mode 100644 Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift diff --git a/Sources/Shared/Publication/Services/Content/Content.swift b/Sources/Shared/Publication/Services/Content/Content.swift index 5618a25c6..e1ed7297d 100644 --- a/Sources/Shared/Publication/Services/Content/Content.swift +++ b/Sources/Shared/Publication/Services/Content/Content.swift @@ -37,9 +37,9 @@ public protocol EmbeddedContentElement: ContentElement { /// An audio clip. public struct AudioContentElement: EmbeddedContentElement, TextualContentElement { - public let locator: Locator - public let embeddedLink: Link - public let attributes: [ContentAttribute] + public var locator: Locator + public var embeddedLink: Link + public var attributes: [ContentAttribute] public init(locator: Locator, embeddedLink: Link, attributes: [ContentAttribute] = []) { self.locator = locator @@ -50,9 +50,9 @@ public struct AudioContentElement: EmbeddedContentElement, TextualContentElement /// A video clip. public struct VideoContentElement: EmbeddedContentElement, TextualContentElement { - public let locator: Locator - public let embeddedLink: Link - public let attributes: [ContentAttribute] + public var locator: Locator + public var embeddedLink: Link + public var attributes: [ContentAttribute] public init(locator: Locator, embeddedLink: Link, attributes: [ContentAttribute] = []) { self.locator = locator @@ -63,12 +63,12 @@ public struct VideoContentElement: EmbeddedContentElement, TextualContentElement /// A bitmap image. public struct ImageContentElement: EmbeddedContentElement, TextualContentElement { - public let locator: Locator - public let embeddedLink: Link + public var locator: Locator + public var embeddedLink: Link /// Short piece of text associated with the image. - public let caption: String? - public let attributes: [ContentAttribute] + public var caption: String? + public var attributes: [ContentAttribute] public init(locator: Locator, embeddedLink: Link, caption: String? = nil, attributes: [ContentAttribute] = []) { self.locator = locator @@ -88,10 +88,10 @@ public struct ImageContentElement: EmbeddedContentElement, TextualContentElement /// @param role Purpose of this element in the broader context of the document. /// @param segments Ranged portions of text with associated attributes. public struct TextContentElement: TextualContentElement { - public let locator: Locator - public let role: Role - public let segments: [Segment] - public let attributes: [ContentAttribute] + public var locator: Locator + public var role: Role + public var segments: [Segment] + public var attributes: [ContentAttribute] public init(locator: Locator, role: Role, segments: [Segment], attributes: [ContentAttribute] = []) { self.locator = locator @@ -125,10 +125,10 @@ public struct TextContentElement: TextualContentElement { /// @param locator Locator to the segment of text. /// @param text Text in the segment. /// @param attributes Attributes associated with this segment, e.g. language. - public struct Segment { - public let locator: Locator - public let text: String - public let attributes: [ContentAttribute] + public struct Segment: ContentAttributesHolder { + public var locator: Locator + public var text: String + public var attributes: [ContentAttribute] public init(locator: Locator, text: String, attributes: [ContentAttribute] = []) { self.locator = locator @@ -155,6 +155,11 @@ public struct ContentAttribute: Hashable { public let key: String public let value: AnyHashable + public init(key: ContentAttributeKey, value: T) { + self.key = key.key + self.value = value + } + public init(key: String, value: AnyHashable) { self.key = key self.value = value @@ -203,15 +208,9 @@ public extension ContentAttributesHolder { /// Iterates through a list of `ContentElement` items. public protocol ContentIterator: AnyObject { - /// Returns true if the iterator has a next element, potentially blocking the caller while processing it. - func hasNext() -> Bool - /// Retrieves the next element, or nil if we reached the end. func next() throws -> ContentElement? - /// Returns true if the iterator has a previous element, potentially blocking the caller while processing it. - func hasPrevious() -> Bool - /// Advances to the previous item and returns it, or null if we reached the beginning. func previous() throws -> ContentElement? } \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/ContentIterationService.swift b/Sources/Shared/Publication/Services/Content/ContentIterationService.swift deleted file mode 100644 index 53a929f0b..000000000 --- a/Sources/Shared/Publication/Services/Content/ContentIterationService.swift +++ /dev/null @@ -1,71 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import Foundation - -public typealias ContentIterationServiceFactory = (PublicationServiceContext) -> ContentIterationService? - -public protocol ContentIterationService: PublicationService { - func iterator(from start: Locator?) -> ContentIteratorOld? -} - -public extension Publication { - var isContentIterable: Bool { - contentIterationService != nil - } - - func contentIterator(from start: Locator?) -> ContentIteratorOld? { - contentIterationService?.iterator(from: start) - } - - private var contentIterationService: ContentIterationService? { - findService(ContentIterationService.self) - } -} - -public extension PublicationServicesBuilder { - mutating func setContentIterationServiceFactory(_ factory: ContentIterationServiceFactory?) { - if let factory = factory { - set(ContentIterationService.self, factory) - } else { - remove(ContentIterationService.self) - } - } -} - -public class DefaultContentIterationService: ContentIterationService { - - public static func makeFactory(resourceContentIteratorFactories: [ResourceContentIteratorFactory]) -> (PublicationServiceContext) -> DefaultContentIterationService? { - { context in - DefaultContentIterationService( - publication: context.publication, - resourceContentIteratorFactories: resourceContentIteratorFactories - ) - } - } - - private let publication: Weak - private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] - - public init( - publication: Weak, - resourceContentIteratorFactories: [ResourceContentIteratorFactory] - ) { - self.publication = publication - self.resourceContentIteratorFactories = resourceContentIteratorFactories - } - - public func iterator(from start: Locator?) -> ContentIteratorOld? { - guard let publication = publication() else { - return nil - } - return PublicationContentIterator( - publication: publication, - start: start, - resourceContentIteratorFactories: resourceContentIteratorFactories - ) - } -} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/ContentIteratorOld.swift b/Sources/Shared/Publication/Services/Content/ContentIteratorOld.swift deleted file mode 100644 index 7a3df4526..000000000 --- a/Sources/Shared/Publication/Services/Content/ContentIteratorOld.swift +++ /dev/null @@ -1,59 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import Foundation - -public struct ContentOld: Equatable { - public let locator: Locator - public let data: Data - - public var extras: [String: Any] { - get { extrasJSON.json } - set { extrasJSON = JSONDictionary(newValue) ?? JSONDictionary() } - } - // Trick to keep the struct equatable despite [String: Any] - private var extrasJSON: JSONDictionary - - public init(locator: Locator, data: Data, extras: [String: Any] = [:]) { - self.locator = locator - self.data = data - self.extrasJSON = JSONDictionary(extras) ?? JSONDictionary() - } - - public enum Data: Equatable { - case audio(target: Link) - case image(target: Link, description: String?) - case text(spans: [TextSpan], style: TextStyle) - } - - public enum TextStyle: Equatable { - case heading(level: Int) - case body - case callout - case caption - case footnote - case quote - case listItem - } - - public struct TextSpan: Equatable { - public let locator: Locator - public let language: Language? - public let text: String - - public init(locator: Locator, language: Language?, text: String) { - self.locator = locator - self.language = language - self.text = text - } - } -} - -public protocol ContentIteratorOld: AnyObject { - func close() - func previous() throws -> ContentOld? - func next() throws -> ContentOld? -} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/ContentService.swift b/Sources/Shared/Publication/Services/Content/ContentService.swift index d5b8cdaa0..4aaac3c1e 100644 --- a/Sources/Shared/Publication/Services/Content/ContentService.swift +++ b/Sources/Shared/Publication/Services/Content/ContentService.swift @@ -37,16 +37,18 @@ public class DefaultContentService: ContentService { guard let pub = publication() else { return nil } - return DefaultContent(publication: pub, start: start) + return DefaultContent(publication: pub, start: start, resourceContentIteratorFactories: resourceContentIteratorFactories) } private class DefaultContent: Content { let publication: Publication let start: Locator? + let resourceContentIteratorFactories: [ResourceContentIteratorFactory] - init(publication: Publication, start: Locator?) { + init(publication: Publication, start: Locator?, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { self.publication = publication self.start = start + self.resourceContentIteratorFactories = resourceContentIteratorFactories } func makeIterator() -> ContentIterator { diff --git a/Sources/Shared/Publication/Services/Content/ContentTokenizer.swift b/Sources/Shared/Publication/Services/Content/ContentTokenizer.swift new file mode 100644 index 000000000..609c47570 --- /dev/null +++ b/Sources/Shared/Publication/Services/Content/ContentTokenizer.swift @@ -0,0 +1,58 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// A tokenizer splitting a `ContentElement` into smaller pieces. +public typealias ContentTokenizer = Tokenizer + +/// A `ContentTokenizer` using a `TextTokenizer` to split the text of the `Content`. +/// +/// - Parameter contextSnippetLength: Length of `before` and `after` snippets in the produced `Locator`s. +public func makeTextContentTokenizer( + defaultLanguage: Language?, + contextSnippetLength: Int = 50, + textTokenizerFactory: @escaping (Language?) -> TextTokenizer +) -> ContentTokenizer { + func tokenize(_ segment: TextContentElement.Segment) throws -> [TextContentElement.Segment] { + let tokenize = textTokenizerFactory(segment.language ?? defaultLanguage) + + return try tokenize(segment.text) + .map { range in + var segment = segment + segment.locator = segment.locator.copy(text: { + $0 = extractTextContext( + in: segment.text, + for: range, + contextSnippetLength: contextSnippetLength + ) + }) + segment.text = String(segment.text[range]) + return segment + } + } + + func tokenize(_ content: ContentElement) throws -> [ContentElement] { + if var content = content as? TextContentElement { + content.segments = try content.segments.flatMap(tokenize) + return [content] + } else { + return [content] + } + } + + return tokenize +} + +private func extractTextContext(in string: String, for range: Range, contextSnippetLength: Int) -> Locator.Text { + let after = String(string[range.upperBound.. ResourceContentIteratorFactory { { resource, locator in - HTMLResourceContentIterator(resource: resource, locator: locator) + guard resource.link.mediaType.isHTML else { + return nil + } + return HTMLResourceContentIterator(resource: resource, locator: locator) } } - private var content: Result<[ContentOld], Error> - private var currentIndex: Int? - private var startingIndex: Int + private let resource: Resource + private let locator: Locator public init(resource: Resource, locator: Locator) { - let result = resource - .readAsString() - .eraseToAnyError() - .tryMap { try SwiftSoup.parse($0) } - .tryMap { document -> (content: [ContentOld], startingIndex: Int) in - try ContentParser.parse(document: document, locator: locator) - } - - content = result.map { $0.content } - startingIndex = result.map { $0.startingIndex }.get(or: 0) + self.resource = resource + self.locator = locator } - public func close() {} - - public func previous() throws -> ContentOld? { + public func previous() throws -> ContentElement? { try next(by: -1) } - public func next() throws -> ContentOld? { + public func next() throws -> ContentElement? { try next(by: +1) } - private func next(by delta: Int) throws -> ContentOld? { - let content = try content.get() - let index = index(by: delta) - guard content.indices.contains(index) else { + private func next(by delta: Int) throws -> ContentElement? { + let elements = try elements.get() + let index = currentIndex.map { $0 + delta } + ?? elements.startIndex + + guard elements.elements.indices.contains(index) else { return nil } + currentIndex = index - return content[index] + return elements.elements[index] } - private func index(by delta: Int) -> Int { - if let i = currentIndex { - return i + delta - } else { - return startingIndex - } + private var currentIndex: Int? + + private lazy var elements: Result = parseElements() + + private func parseElements() -> Result { + let result = resource + .readAsString() + .eraseToAnyError() + .tryMap { try SwiftSoup.parse($0) } + .tryMap { try ContentParser.parse(document: $0, locator: locator) } + resource.close() + return result } + + /// Holds the result of parsing the HTML resource into a list of `ContentElement`. + /// + /// The `startIndex` will be calculated from the element matched by the base `locator`, if possible. Defaults to + /// 0. + private typealias ParsedElements = (elements: [ContentElement], startIndex: Int) + private class ContentParser: NodeVisitor { - static func parse(document: Document, locator: Locator) throws -> (content: [ContentOld], startingIndex: Int) { + static func parse(document: Document, locator: Locator) throws -> ParsedElements { let parser = ContentParser( baseLocator: locator, startElement: try locator.locations.cssSelector @@ -73,27 +89,25 @@ public class HTMLResourceContentIterator : ContentIteratorOld { try document.select($0.removingPrefix(":root > ")).first() } ) + try document.traverse(parser) - - var result = ( - content: parser.content, - startingIndex: parser.startIndex + + return ParsedElements( + elements: parser.elements, + startIndex: (locator.locations.progression == 1.0) + ? parser.elements.count - 1 + : parser.startIndex ) - - if locator.locations.progression == 1.0 { - result.startingIndex = result.content.count - 1 - } - - return result } private let baseLocator: Locator private let startElement: Element? - private(set) var content: [ContentOld] = [] - private(set) var startIndex = 0 + private var elements: [ContentElement] = [] + private var startIndex = 0 private var currentElement: Element? - private var spansAcc: [ContentOld.TextSpan] = [] + + private var segmentsAcc: [TextContentElement.Segment] = [] private var textAcc = StringBuilder() private var wholeRawTextAcc: String = "" private var elementRawTextAcc: String = "" @@ -119,7 +133,6 @@ public class HTMLResourceContentIterator : ContentIteratorOld { if let elem = node as? Element { currentElement = elem - let cssSelector = try elem.cssSelector() let tag = elem.tagNameNormal() if tag == "br" { @@ -127,29 +140,35 @@ public class HTMLResourceContentIterator : ContentIteratorOld { } else if tag == "img" { flushText() - if let href = try elem.attr("src") - .takeUnlessEmpty() - .map({ HREF($0, relativeTo: baseLocator.href).string }) { - content.append(ContentOld( + if + let href = try elem.attr("src") + .takeUnlessEmpty() + .map({ HREF($0, relativeTo: baseLocator.href).string }) + { + var attributes: [ContentAttribute] = [] + if let alt = try elem.attr("alt").takeUnlessEmpty() { + attributes.append(ContentAttribute(key: .accessibilityLabel, value: alt)) + } + + elements.append(ImageContentElement( locator: baseLocator.copy( locations: { $0 = Locator.Locations( - otherLocations: ["cssSelector": cssSelector] + otherLocations: ["cssSelector": try? elem.cssSelector()] ) } ), - data: .image( - target: Link(href: href), - description: try elem.attr("alt").takeUnlessEmpty() - ) + embeddedLink: Link(href: href), + caption: nil, // FIXME: Get the caption from figcaption + attributes: attributes )) } } else if elem.isBlock() { - spansAcc.removeAll() + segmentsAcc.removeAll() textAcc.clear() rawTextAcc = "" - currentCSSSelector = cssSelector + currentCSSSelector = try elem.cssSelector() } } } @@ -162,7 +181,7 @@ public class HTMLResourceContentIterator : ContentIteratorOld { if let node = node as? TextNode { let language = try node.language().map { Language(code: .bcp47($0)) } if (currentLanguage != language) { - flushSpan() + flushSegment() currentLanguage = language } @@ -186,15 +205,15 @@ public class HTMLResourceContentIterator : ContentIteratorOld { } private func flushText() { - flushSpan() - guard !spansAcc.isEmpty else { + flushSegment() + guard !segmentsAcc.isEmpty else { return } if startElement != nil && currentElement == startElement { - startIndex = content.count + startIndex = elements.count } - content.append(ContentOld( + elements.append(TextContentElement( locator: baseLocator.copy( locations: { [self] in $0 = Locator.Locations( @@ -209,18 +228,19 @@ public class HTMLResourceContentIterator : ContentIteratorOld { ) } ), - data: .text(spans: spansAcc, style: .body) + role: .body, + segments: segmentsAcc )) elementRawTextAcc = "" - spansAcc.removeAll() + segmentsAcc.removeAll() } - private func flushSpan() { + private func flushSegment() { var text = textAcc.toString() let trimmedText = text.trimmingCharacters(in: .whitespacesAndNewlines) if !text.isEmpty { - if spansAcc.isEmpty { + if segmentsAcc.isEmpty { let whitespaceSuffix = text.last .takeIf { $0.isWhitespace || $0.isNewline } .map { String($0) } @@ -229,7 +249,12 @@ public class HTMLResourceContentIterator : ContentIteratorOld { text = trimmedText + whitespaceSuffix } - spansAcc.append(ContentOld.TextSpan( + var attributes: [ContentAttribute] = [] + if let lang = currentLanguage { + attributes.append(ContentAttribute(key: .language, value: lang)) + } + + segmentsAcc.append(TextContentElement.Segment( locator: baseLocator.copy( locations: { [self] in $0 = Locator.Locations( @@ -245,8 +270,8 @@ public class HTMLResourceContentIterator : ContentIteratorOld { ) } ), - language: currentLanguage, - text: text + text: text, + attributes: attributes )) } diff --git a/Sources/Shared/Publication/Services/Content/Iterators/PublicationContentIterator.swift b/Sources/Shared/Publication/Services/Content/Iterators/PublicationContentIterator.swift new file mode 100644 index 000000000..8f26e38d2 --- /dev/null +++ b/Sources/Shared/Publication/Services/Content/Iterators/PublicationContentIterator.swift @@ -0,0 +1,151 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// Creates a `ContentIterator` instance for the `resource`, starting from the given `locator`. +/// +/// - Returns: nil if the resource format is not supported. +public typealias ResourceContentIteratorFactory = + (_ resource: Resource, _ locator: Locator) -> ContentIterator? + +/// A composite [Content.Iterator] which iterates through a whole [publication] and delegates the +/// iteration inside a given resource to media type-specific iterators. +public class PublicationContentIterator: ContentIterator, Loggable { + + /// `ContentIterator` for a resource, associated with its index in the reading order. + private typealias IndexedIterator = (index: Int, iterator: ContentIterator) + + private enum Direction: Int { + case forward = 1 + case backward = -1 + } + + private let publication: Publication + private var startLocator: Locator? + private var _currentIterator: IndexedIterator? + + /// List of `ResourceContentIteratorFactory` which will be used to create the iterator for each resource. The + /// factories are tried in order until there's a match. + private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] + + public init(publication: Publication, start: Locator?, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { + self.publication = publication + self.startLocator = start + self.resourceContentIteratorFactories = resourceContentIteratorFactories + } + + public func previous() throws -> ContentElement? { + try next(.backward) + } + + public func next() throws -> ContentElement? { + try next(.forward) + } + + private func next(_ direction: Direction) throws -> ContentElement? { + guard let iterator = currentIterator else { + return nil + } + + let content: ContentElement? = try { + switch direction { + case .forward: + return try iterator.iterator.next() + case .backward: + return try iterator.iterator.previous() + } + }() + guard content != nil else { + guard let nextIterator = nextIterator(direction, fromIndex: iterator.index) else { + return nil + } + _currentIterator = nextIterator + return try next(direction) + } + + return content + } + + /// Returns the `ContentIterator` for the current `Resource` in the reading order. + private var currentIterator: IndexedIterator? { + if _currentIterator == nil { + _currentIterator = initialIterator() + } + return _currentIterator + } + + /// Returns the first iterator starting at `startLocator` or the beginning of the publication. + private func initialIterator() -> IndexedIterator? { + let index = startLocator.flatMap { publication.readingOrder.firstIndex(withHREF: $0.href) } ?? 0 + let location = startLocator.orProgression(0.0) + + return loadIterator(at: index, location: location) + ?? nextIterator(.forward, fromIndex: index) + } + + /// Returns the next resource iterator in the given `direction`, starting from `fromIndex`. + private func nextIterator(_ direction: Direction, fromIndex: Int) -> IndexedIterator? { + let index = fromIndex + direction.rawValue + guard publication.readingOrder.indices.contains(index) else { + return nil + } + + let progression: Double = { + switch direction { + case .forward: + return 0.0 + case .backward: + return 1.0 + } + }() + + return loadIterator(at: index, location: .progression(progression)) + ?? nextIterator(direction, fromIndex: index) + } + + /// Loads the iterator at the given `index` in the reading order. + /// + /// The `location` will be used to compute the starting `Locator` for the iterator. + private func loadIterator(at index: Int, location: LocatorOrProgression) -> IndexedIterator? { + let link = publication.readingOrder[index] + guard let locator = location.toLocator(to: link, in: publication) else { + return nil + } + + let resource = publication.get(link) + return resourceContentIteratorFactories + .first { factory in factory(resource, locator) } + .map { IndexedIterator(index: index, iterator: $0) } + } +} + +/// Represents either a full `Locator`, or a progression percentage in a resource. +private enum LocatorOrProgression { + case locator(Locator) + case progression(Double) + + func toLocator(to link: Link, in publication: Publication) -> Locator? { + switch self { + case .locator(let locator): + return locator + case .progression(let progression): + return publication.locate(link)?.copy(locations: { $0.progression = progression }) + } + } +} + +private extension Optional where Wrapped == Locator { + + /// Returns this locator if not null, or the given `progression` as a fallback. + func orProgression(_ progression: Double) -> LocatorOrProgression { + if case let .some(locator) = self { + return .locator(locator) + } else { + return .progression(progression) + } + } +} \ No newline at end of file diff --git a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift b/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift deleted file mode 100644 index 80c5c2c28..000000000 --- a/Sources/Shared/Publication/Services/Content/PublicationContentIterator.swift +++ /dev/null @@ -1,124 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import Foundation - -/// Creates a `ContentIterator` instance for the given `resource`. -/// -/// - Returns: nil if the resource format is not supported. -public typealias ResourceContentIteratorFactory = - (_ resource: Resource, _ locator: Locator) -> ContentIteratorOld? - -public class PublicationContentIterator: ContentIteratorOld, Loggable { - - private let publication: Publication - private var startLocator: Locator? - private let resourceContentIteratorFactories: [ResourceContentIteratorFactory] - private var startIndex: Int? - private var currentIndex: Int = 0 - private var currentIterator: ContentIteratorOld? - - public init(publication: Publication, start: Locator?, resourceContentIteratorFactories: [ResourceContentIteratorFactory]) { - self.publication = publication - self.startLocator = start - self.resourceContentIteratorFactories = resourceContentIteratorFactories - - startIndex = { - guard - let start = start, - let index = publication.readingOrder.firstIndex(withHREF: start.href) - else { - return 0 - } - return index - }() - } - - public func close() { - currentIterator?.close() - currentIterator = nil - } - - public func previous() throws -> ContentOld? { - guard let iterator = iterator(by: -1) else { - return nil - } - guard let content = try iterator.previous() else { - currentIterator = nil - return try previous() - } - return content - } - - public func next() throws -> ContentOld? { - guard let iterator = iterator(by: +1) else { - return nil - } - guard let content = try iterator.next() else { - currentIterator = nil - return try next() - } - return content - } - - private func iterator(by delta: Int) -> ContentIteratorOld? { - if let iter = currentIterator { - return iter - } - - // For the first requested iterator, we don't want to move by the given delta. - var delta = delta - if let start = startIndex { - startIndex = nil - currentIndex = start - delta = 0 - } - - guard let (newIndex, newIterator) = loadIterator(from: currentIndex, by: delta) else { - return nil - } - currentIndex = newIndex - currentIterator = newIterator - return newIterator - } - - private func loadIterator(from index: Int, by delta: Int) -> (index: Int, ContentIteratorOld)? { - let i = index + delta - guard - let link = publication.readingOrder.getOrNil(i), - var locator = publication.locate(link) - else { - return nil - } - - if let start = startLocator.pop() { - locator = locator.copy( - locations: { $0 = start.locations }, - text: { $0 = start.text } - ) - } else if delta < 0 { - locator = locator.copy( - locations: { $0.progression = 1.0 } - ) - } - - guard let iterator = loadIterator(at: link, locator: locator) else { - return loadIterator(from: i, by: delta) - } - return (i, iterator) - } - - private func loadIterator(at link: Link, locator: Locator) -> ContentIteratorOld? { - let resource = publication.get(link) - for factory in resourceContentIteratorFactories { - if let iterator = factory(resource, locator) { - return iterator - } - } - - return nil - } -} diff --git a/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift b/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift index 117f4ba7b..b6ed896a1 100644 --- a/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift +++ b/Sources/Shared/Publication/Services/PublicationServicesBuilder.swift @@ -14,7 +14,7 @@ public struct PublicationServicesBuilder { private var factories: [String: PublicationServiceFactory] = [:] public init( - contentIteration: ContentIterationServiceFactory? = nil, + content: ContentServiceFactory? = nil, contentProtection: ContentProtectionServiceFactory? = nil, cover: CoverServiceFactory? = nil, locator: LocatorServiceFactory? = { DefaultLocatorService(publication: $0.publication) }, @@ -22,7 +22,7 @@ public struct PublicationServicesBuilder { search: SearchServiceFactory? = nil, setup: (inout PublicationServicesBuilder) -> Void = { _ in } ) { - setContentIterationServiceFactory(contentIteration) + setContentServiceFactory(content) setContentProtectionServiceFactory(contentProtection) setCoverServiceFactory(cover) setLocatorServiceFactory(locator) diff --git a/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift b/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift deleted file mode 100644 index c9a776ccb..000000000 --- a/Sources/Shared/Toolkit/Tokenizer/ContentTokenizer.swift +++ /dev/null @@ -1,56 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import Foundation - -/// A tokenizer splitting a `Content` into smaller pieces. -public typealias ContentTokenizer = Tokenizer - -/// A `ContentTokenizer` using the default `TextTokenizer` to split the text of the `Content` by `unit`. -public func makeTextContentTokenizer(unit: TextUnit, language: Language?) -> ContentTokenizer { - makeTextContentTokenizer(with: makeDefaultTextTokenizer(unit: unit, language: language)) -} - -/// A `ContentTokenizer` using a `TextTokenizer` to split the text of the `Content`. -public func makeTextContentTokenizer(with tokenizer: @escaping TextTokenizer) -> ContentTokenizer { - func tokenize(_ span: ContentOld.TextSpan) throws -> [ContentOld.TextSpan] { - try tokenizer(span.text) - .map { range in - ContentOld.TextSpan( - locator: span.locator.copy(text: { $0 = extractTextContext(in: span.text, for: range) }), - language: span.language, - text: String(span.text[range]) - ) - } - } - - func tokenize(_ content: ContentOld) throws -> [ContentOld] { - switch content.data { - case .audio, .image: - return [content] - case .text(spans: let spans, style: let style): - return [ContentOld( - locator: content.locator, - data: .text( - spans: try spans.flatMap { try tokenize($0) }, - style: style - ) - )] - } - } - - return tokenize -} - -private func extractTextContext(in string: String, for range: Range) -> Locator.Text { - let after = String(string[range.upperBound.. Date: Sun, 7 Aug 2022 14:04:52 +0200 Subject: [PATCH 33/46] Refactor `firstVisibleElementLocator` --- .../EPUB/EPUBNavigatorViewController.swift | 2 +- Sources/Navigator/Navigator.swift | 9 --------- Sources/Navigator/VisualNavigator.swift | 13 +++++++++---- .../Sources/Reader/Common/TTS/TTSViewModel.swift | 2 +- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift b/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift index 6b426d572..fb50c6349 100644 --- a/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift +++ b/Sources/Navigator/EPUB/EPUBNavigatorViewController.swift @@ -526,7 +526,7 @@ open class EPUBNavigatorViewController: UIViewController, VisualNavigator, Selec } } - public func findLocationOfFirstVisibleContent(completion: @escaping (Locator?) -> ()) { + public func firstVisibleElementLocator(completion: @escaping (Locator?) -> ()) { guard let spreadView = paginationView.currentView as? EPUBSpreadView else { DispatchQueue.main.async { completion(nil) } return diff --git a/Sources/Navigator/Navigator.swift b/Sources/Navigator/Navigator.swift index 7d7b62ad0..baaccba29 100644 --- a/Sources/Navigator/Navigator.swift +++ b/Sources/Navigator/Navigator.swift @@ -20,9 +20,6 @@ public protocol Navigator { /// Can be used to save a bookmark to the current position. var currentLocation: Locator? { get } - /// Returns a `Locator` targeting the first visible content element on the current resource. - func findLocationOfFirstVisibleContent(completion: @escaping (Locator?) -> Void) - /// Moves to the position in the publication correponding to the given `Locator`. /// - Parameter completion: Called when the transition is completed. /// - Returns: Whether the navigator is able to move to the locator. The completion block is only called if true was returned. @@ -51,12 +48,6 @@ public protocol Navigator { public extension Navigator { - func findLocationOfFirstVisibleContent(completion: @escaping (Locator?) -> ()) { - DispatchQueue.main.async { - completion(nil) - } - } - /// Adds default values for the parameters. @discardableResult func go(to locator: Locator, animated: Bool = false, completion: @escaping () -> Void = {}) -> Bool { diff --git a/Sources/Navigator/VisualNavigator.swift b/Sources/Navigator/VisualNavigator.swift index 9dfc7245e..9ac820a2a 100644 --- a/Sources/Navigator/VisualNavigator.swift +++ b/Sources/Navigator/VisualNavigator.swift @@ -29,10 +29,18 @@ public protocol VisualNavigator: Navigator { @discardableResult func goRight(animated: Bool, completion: @escaping () -> Void) -> Bool + /// Returns the `Locator` to the first content element that begins on the current screen. + func firstVisibleElementLocator(completion: @escaping (Locator?) -> Void) } public extension VisualNavigator { - + + func firstVisibleElementLocator(completion: @escaping (Locator?) -> ()) { + DispatchQueue.main.async { + completion(currentLocation) + } + } + @discardableResult func goLeft(animated: Bool = false, completion: @escaping () -> Void = {}) -> Bool { switch readingProgression { @@ -52,7 +60,6 @@ public extension VisualNavigator { return goBackward(animated: animated, completion: completion) } } - } @@ -61,7 +68,6 @@ public protocol VisualNavigatorDelegate: NavigatorDelegate { /// Called when the user tapped the publication, and it didn't trigger any internal action. /// The point is relative to the navigator's view. func navigator(_ navigator: VisualNavigator, didTapAt point: CGPoint) - } public extension VisualNavigatorDelegate { @@ -69,5 +75,4 @@ public extension VisualNavigatorDelegate { func navigator(_ navigator: VisualNavigator, didTapAt point: CGPoint) { // Optional } - } diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index c84d9f2e7..d03ca95ae 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -70,7 +70,7 @@ final class TTSViewModel: ObservableObject, Loggable { .sorted { $0.localizedDescription() < $1.localizedDescription() } @objc func play() { - navigator.findLocationOfFirstVisibleContent { [self] locator in + navigator.firstVisibleElementLocator { [self] locator in tts.play(from: locator ?? navigator.currentLocation) } } From 862d89dce22c2249a4f8d3aa7bee91adccb7c734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sun, 7 Aug 2022 16:08:41 +0200 Subject: [PATCH 34/46] Refactor TTSEngine --- Sources/Navigator/TTS/AVTTSEngine.swift | 327 +++++++++++----------- Sources/Navigator/TTS/TTSController.swift | 2 +- Sources/Navigator/TTS/TTSEngine.swift | 98 ++++--- Sources/Shared/Toolkit/Cancellable.swift | 13 +- 4 files changed, 237 insertions(+), 203 deletions(-) diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index 01eae0a03..b371b4e38 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -27,21 +27,22 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier private let avPitchRange = 0.5...2.0 - public let defaultConfig: TTSConfiguration - public var config: TTSConfiguration - private let debug: Bool - - public weak var delegate: TTSEngineDelegate? + /// Conversion function between a rate multiplier and the `AVSpeechUtterance` rate. + private let rateMultiplierToAVRate: (Double) -> Float + private let debug: Bool private let synthesizer = AVSpeechSynthesizer() /// Creates a new `AVTTSEngine` instance. /// /// - Parameters: + /// - rateMultiplierToAVRate: Conversion function between a rate multiplier and the `AVSpeechUtterance` rate. /// - audioSessionConfig: AudioSession configuration used while playing utterances. If `nil`, utterances won't /// play when the app is in the background. /// - debug: Print the state machine transitions. public init( + // FIXME: + rateMultiplierToAVRate: @escaping (Double) -> Float = { Float($0) }, audioSessionConfig: _AudioSession.Configuration? = .init( category: .playback, mode: .spokenAudio, @@ -49,21 +50,17 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg ), debug: Bool = false ) { - let config = TTSConfiguration( - defaultLanguage: Language(code: .bcp47(AVSpeechSynthesisVoice.currentLanguageCode())), - rate: avRateRange.percentageForValue(Double(AVSpeechUtteranceDefaultSpeechRate)), - pitch: avPitchRange.percentageForValue(1.0) - ) - - self.defaultConfig = config - self.config = config - self.debug = debug + self.rateMultiplierToAVRate = rateMultiplierToAVRate self.audioSessionUser = audioSessionConfig.map { AudioSessionUser(config: $0) } + self.debug = debug super.init() synthesizer.delegate = self } + // FIXME: Double check + public let rateMultiplierRange: ClosedRange = 0.5...2.0 + public lazy var availableVoices: [TTSVoice] = AVSpeechSynthesisVoice.speechVoices() .map { TTSVoice(voice: $0) } @@ -73,57 +70,92 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg .map { TTSVoice(voice: $0) } } - public func speak(_ utterance: TTSUtterance) { - on(.play(utterance)) + public func speak( + _ utterance: TTSUtterance, + onSpeakRange: @escaping (Range) -> (), + completion: @escaping (Result) -> () + ) -> Cancellable { + + let task = Task( + utterance: utterance, + onSpeakRange: onSpeakRange, + completion: completion + ) + let cancellable = CancellableObject { [weak self] in + self?.on(.stop(task)) + } + task.cancellable = cancellable + + on(.play(task)) + + return cancellable } - public func stop() { - on(.stop) + private class Task: Equatable { + let utterance: TTSUtterance + let onSpeakRange: (Range) -> () + let completion: (Result) -> () + var cancellable: CancellableObject? = nil + + init(utterance: TTSUtterance, onSpeakRange: @escaping (Range) -> (), completion: @escaping (Result) -> ()) { + self.utterance = utterance + self.onSpeakRange = onSpeakRange + self.completion = completion + } + + static func ==(lhs: Task, rhs: Task) -> Bool { + ObjectIdentifier(lhs) == ObjectIdentifier(rhs) + } } - - + + private func taskUtterance(with task: Task) -> TaskUtterance { + let utter = TaskUtterance(task: task) + utter.rate = rateMultiplierToAVRate(task.utterance.rateMultiplier) + // FIXME: +// utter.pitchMultiplier = Float(avPitchRange.valueForPercentage(config.pitch)) + utter.preUtteranceDelay = task.utterance.delay + utter.voice = voice(for: task.utterance) + return utter + } + + private class TaskUtterance: AVSpeechUtterance { + let task: Task + + init(task: Task) { + self.task = task + super.init(string: task.utterance.text) + } + + required init?(coder: NSCoder) { + fatalError("Not supported") + } + } + // MARK: AVSpeechSynthesizerDelegate public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didStart utterance: AVSpeechUtterance) { - guard let utterance = (utterance as? AVUtterance)?.utterance else { + guard let task = (utterance as? TaskUtterance)?.task else { return } - on(.didStart(utterance)) + on(.didStart(task)) } public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { - guard let utterance = (utterance as? AVUtterance)?.utterance else { + guard let task = (utterance as? TaskUtterance)?.task else { return } - on(.didFinish(utterance)) + on(.didFinish(task)) } public func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, willSpeakRangeOfSpeechString characterRange: NSRange, utterance avUtterance: AVSpeechUtterance) { guard - let utterance = (avUtterance as? AVUtterance)?.utterance, - let highlight = utterance.locator.text.highlight, - let range = Range(characterRange, in: highlight) + let task = (avUtterance as? TaskUtterance)?.task, + let range = Range(characterRange, in: task.utterance.text) else { return } - let rangeLocator = utterance.locator.copy( - text: { text in text = text[range] } - ) - on(.willSpeakRange(locator: rangeLocator, utterance: utterance)) - } - - private class AVUtterance: AVSpeechUtterance { - let utterance: TTSUtterance - - init(utterance: TTSUtterance) { - self.utterance = utterance - super.init(string: utterance.text) - } - - required init?(coder: NSCoder) { - fatalError("Not supported") - } + on(.willSpeakRange(range, task: task)) } @@ -164,108 +196,27 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// The TTS engine is waiting for the next utterance to play. case stopped /// A new utterance is being processed by the TTS engine, we wait for didStart. - case starting(TTSUtterance) + case starting(Task) /// The utterance is currently playing and the engine is ready to process other commands. - case playing(TTSUtterance) + case playing(Task) /// The engine was stopped while processing the previous utterance, we wait for didStart /// and/or didFinish. The queued utterance will be played once the engine is successfully stopped. - case stopping(TTSUtterance, queued: TTSUtterance?) - - mutating func on(_ event: Event) -> Effect? { - switch (self, event) { - - // stopped - - case let (.stopped, .play(utterance)): - self = .starting(utterance) - return .play(utterance) - - // starting - - case let (.starting(current), .didStart(started)) where current == started: - self = .playing(current) - return nil - - case let (.starting(current), .play(next)): - self = .stopping(current, queued: next) - return nil - - case let (.starting(current), .stop): - self = .stopping(current, queued: nil) - return nil - - // playing - - case let (.playing(current), .didFinish(finished)) where current == finished: - self = .stopped - return .notifyDidStopAfterLastUtterance(current) - - case let (.playing(current), .play(next)): - self = .stopping(current, queued: next) - return .stop - - case let (.playing(current), .stop): - self = .stopping(current, queued: nil) - return .stop - - case let (.playing(current), .willSpeakRange(locator: Locator, utterance: speaking)) where current == speaking: - return .notifyWillSpeakRange(locator: Locator, utterance: current) - - // stopping - - case let (.stopping(current, queued: next), .didStart(started)) where current == started: - self = .stopping(current, queued: next) - return .stop - - case let (.stopping(current, queued: next), .didFinish(finished)) where current == finished: - if let next = next { - self = .starting(next) - return .play(next) - } else { - self = .stopped - return .notifyDidStopAfterLastUtterance(current) - } - - case let (.stopping(current, queued: _), .play(next)): - self = .stopping(current, queued: next) - return nil - - case let (.stopping(current, queued: _), .stop): - self = .stopping(current, queued: nil) - return nil - - - default: - return nil - } - } + case stopping(Task, queued: Task?) } /// State machine events triggered by the `AVSpeechSynthesizer` or the client /// of `AVTTSEngine`. private enum Event: Equatable { // AVTTSEngine commands - case play(TTSUtterance) - case stop + case play(Task) + case stop(Task) // AVSpeechSynthesizer delegate events - case didStart(TTSUtterance) - case willSpeakRange(locator: Locator, utterance: TTSUtterance) - case didFinish(TTSUtterance) + case didStart(Task) + case willSpeakRange(Range, task: Task) + case didFinish(Task) } - - /// State machine side effects triggered by a state transition from an event. - private enum Effect: Equatable { - // Ask `AVSpeechSynthesizer` to play the utterance. - case play(TTSUtterance) - // Ask `AVSpeechSynthesizer` to stop the playback. - case stop - - // Send notifications to our delegate. - case notifyWillSpeakRange(locator: Locator, utterance: TTSUtterance) - case notifyDidStopAfterLastUtterance(TTSUtterance) - } - + private var state: State = .stopped { didSet { if (debug) { @@ -281,49 +232,87 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg if (debug) { log(.debug, "-> on \(event)") } - - if let effect = state.on(event) { - handle(effect) + + switch (state, event) { + + // stopped + + case let (.stopped, .play(task)): + state = .starting(task) + startEngine(with: task) + + // starting + + case let (.starting(current), .didStart(started)) where current == started: + state = .playing(current) + + case let (.starting(current), .play(next)): + state = .stopping(current, queued: next) + + case let (.starting(current), .stop): + state = .stopping(current, queued: nil) + + // playing + + case let (.playing(current), .didFinish(finished)) where current == finished: + current.completion(.success(())) + state = .stopped + + case let (.playing(current), .play(next)): + stopEngine() + state = .stopping(current, queued: next) + + case let (.playing(current), .stop): + stopEngine() + state = .stopping(current, queued: nil) + + case let (.playing(current), .willSpeakRange(range, task: speaking)) where current == speaking: + current.onSpeakRange(range) + + // stopping + + case let (.stopping(current, queued: next), .didStart(started)) where current == started: + stopEngine() + state = .stopping(current, queued: next) + + case let (.stopping(current, queued: next), .didFinish(finished)) where current == finished: + if let next = next { + startEngine(with: next) + state = .starting(next) + } else { + current.completion(.success(())) + state = .stopped + } + + case let (.stopping(current, queued: _), .play(next)): + state = .stopping(current, queued: next) + + case let (.stopping(current, queued: _), .stop): + state = .stopping(current, queued: nil) + + + default: + break } } - - /// Handles a state machine side effect. - private func handle(_ effect: Effect) { - switch effect { - - case let .play(utterance): - synthesizer.speak(avUtterance(from: utterance)) - - if let user = audioSessionUser { - _AudioSession.shared.start(with: user) - } - case .stop: - synthesizer.stopSpeaking(at: .immediate) - - case let .notifyWillSpeakRange(locator: Locator, utterance: utterance): - delegate?.ttsEngine(self, willSpeakRangeAt: Locator, of: utterance) - - case let .notifyDidStopAfterLastUtterance(utterance): - delegate?.ttsEngine(self, didStopAfterLastUtterance: utterance) + private func startEngine(with task: Task) { + synthesizer.speak(taskUtterance(with: task)) + + if let user = audioSessionUser { + _AudioSession.shared.start(with: user) } } - - private func avUtterance(from utterance: TTSUtterance) -> AVSpeechUtterance { - let avUtterance = AVUtterance(utterance: utterance) - avUtterance.rate = Float(avRateRange.valueForPercentage(config.rate)) - avUtterance.pitchMultiplier = Float(avPitchRange.valueForPercentage(config.pitch)) - avUtterance.preUtteranceDelay = utterance.delay - avUtterance.postUtteranceDelay = config.delay - avUtterance.voice = voice(for: utterance) - return avUtterance + + private func stopEngine() { + synthesizer.stopSpeaking(at: .immediate) } - + private func voice(for utterance: TTSUtterance) -> AVSpeechSynthesisVoice? { - let language = utterance.language ?? config.defaultLanguage - if let voice = config.voice, voice.language.removingRegion() == language.removingRegion() { + switch utterance.voiceOrLanguage { + case .left(let voice): return AVSpeechSynthesisVoice(identifier: voice.identifier) - } else { + case .right(let language): return AVSpeechSynthesisVoice(language: language) } } diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift index ef9d548ae..ac7916786 100644 --- a/Sources/Navigator/TTS/TTSController.swift +++ b/Sources/Navigator/TTS/TTSController.swift @@ -22,7 +22,7 @@ public extension TTSControllerDelegate { func ttsController(_ ttsController: TTSController, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) {} } -public struct TTSUtterance: Equatable { +public struct TTSUtterance2: Equatable { public let text: String public let locator: Locator public let language: Language? diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index 03975a145..cdd53f0bc 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -7,49 +7,73 @@ import Foundation import R2Shared +/// A text-to-speech engine synthesizes text utterances (e.g. sentence). +/// +/// Implement this interface to support third-party engines with `PublicationSpeechSynthesizer`. public protocol TTSEngine: AnyObject { - var defaultConfig: TTSConfiguration { get } - var config: TTSConfiguration { get set } - var delegate: TTSEngineDelegate? { get set } + + /// Supported range for the speech rate multiplier. + var rateMultiplierRange: ClosedRange { get } + + /// List of available synthesizer voices. var availableVoices: [TTSVoice] { get } - func voiceWithIdentifier(_ id: String) -> TTSVoice? - func speak(_ utterance: TTSUtterance) - func stop() -} + /// Returns the voice with given identifier, if it exists. + func voiceWithIdentifier(_ identifier: String) -> TTSVoice? -public protocol TTSEngineDelegate: AnyObject { - func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) - func ttsEngine(_ engine: TTSEngine, didStopAfterLastUtterance utterance: TTSUtterance) + /// Synthesizes the given `utterance` and returns its status. + /// + /// `onSpeakRange` is called repeatedly while the engine plays portions (e.g. words) of the utterance. + func speak( + _ utterance: TTSUtterance, + onSpeakRange: @escaping (Range) -> Void, + completion: @escaping (Result) -> Void + ) -> Cancellable } -public struct TTSConfiguration { - public var defaultLanguage: Language { - didSet { - defaultLanguage = defaultLanguage.removingRegion() - voice = nil - } +public extension TTSEngine { + func voiceWithIdentifier(_ identifier: String) -> TTSVoice? { + availableVoices.first { $0.identifier == identifier } } - public var rate: Double - public var pitch: Double - public var voice: TTSVoice? +} + +public enum TTSError: Error { + /// Tried to synthesize an utterance with an unsupported language. + case languageNotSupported(language: Language, cause: Error?) + + /// Other engine-specific errors. + case other(Error) +} + +/// An utterance is an arbitrary text (e.g. sentence) that can be synthesized by the TTS engine. +public struct TTSUtterance { + /// Text to be spoken. + public let text: String + + /// Delay before speaking the utterance, in seconds. public var delay: TimeInterval - public init( - defaultLanguage: Language, - rate: Double = 0.5, - pitch: Double = 0.5, - voice: TTSVoice? = nil, - delay: TimeInterval = 0 - ) { - self.defaultLanguage = defaultLanguage.removingRegion() - self.rate = rate - self.pitch = pitch - self.voice = voice - self.delay = delay + /// Multiplier for the speech rate. + public let rateMultiplier: Double + + /// Voice pitch. + public var pitch: Double + + /// Either an explicit voice or the language of the text. If a language is provided, the default voice for this + /// language will be used. + public let voiceOrLanguage: Either + + public var language: Language { + switch voiceOrLanguage { + case .left(let voice): + return voice.language + case .right(let language): + return language + } } } +/// Represents a voice provided by the TTS engine which can speak an utterance. public struct TTSVoice: Hashable { public enum Gender: Hashable { case female, male, unspecified @@ -59,10 +83,20 @@ public struct TTSVoice: Hashable { case low, medium, high } + /// Unique and stable identifier for this voice. Can be used to store and retrieve the voice from the user + /// preferences. public let identifier: String + + /// Human-friendly name for this voice, when available. + public let name: String? + + /// Language (and region) this voice belongs to. public let language: Language - public let name: String + + /// Voice gender. public let gender: Gender + + /// Voice quality. public let quality: Quality? public init(identifier: String, language: Language, name: String, gender: Gender, quality: Quality?) { diff --git a/Sources/Shared/Toolkit/Cancellable.swift b/Sources/Shared/Toolkit/Cancellable.swift index bf3bb119c..e509f4551 100644 --- a/Sources/Shared/Toolkit/Cancellable.swift +++ b/Sources/Shared/Toolkit/Cancellable.swift @@ -14,10 +14,21 @@ public protocol Cancellable { /// A `Cancellable` object saving its cancelled state. public final class CancellableObject: Cancellable { + public private(set) var isCancelled = false + private let onCancel: () -> Void + + public init(onCancel: @escaping () -> Void = {}) { + self.onCancel = onCancel + } public func cancel() { + guard !isCancelled else { + return + } + isCancelled = true + onCancel() } } @@ -70,4 +81,4 @@ public extension Cancellable { func mediated(by mediator: MediatorCancellable) { mediator.mediate(self) } -} \ No newline at end of file +} From e855e5475f7a778d377617cd99b77892e6b5cb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sun, 7 Aug 2022 18:13:41 +0200 Subject: [PATCH 35/46] Add `PublicationSpeechSynthesizer` --- Sources/Navigator/TTS/AVTTSEngine.swift | 10 +- .../TTS/PublicationSpeechSynthesizer.swift | 369 ++++++++++++++++++ Sources/Navigator/TTS/TTSEngine.swift | 7 +- Sources/Navigator/Toolkit/CursorList.swift | 43 ++ 4 files changed, 426 insertions(+), 3 deletions(-) create mode 100644 Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift create mode 100644 Sources/Navigator/Toolkit/CursorList.swift diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index b371b4e38..f3b1b0dc0 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -58,6 +58,10 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg synthesizer.delegate = self } + public func close() { + on(.close) + } + // FIXME: Double check public let rateMultiplierRange: ClosedRange = 0.5...2.0 @@ -207,10 +211,12 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// State machine events triggered by the `AVSpeechSynthesizer` or the client /// of `AVTTSEngine`. private enum Event: Equatable { + case close + // AVTTSEngine commands case play(Task) case stop(Task) - + // AVSpeechSynthesizer delegate events case didStart(Task) case willSpeakRange(Range, task: Task) @@ -236,6 +242,8 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg switch (state, event) { // stopped + case (_, .close): + stopEngine() case let (.stopped, .play(task)): state = .starting(task) diff --git a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift new file mode 100644 index 000000000..f997ed16b --- /dev/null +++ b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift @@ -0,0 +1,369 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation +import R2Shared + +public protocol PublicationSpeechSynthesizerDelegate: AnyObject { + /// Called when the synthesizer's configuration is updated. + func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, configDidChange config: PublicationSpeechSynthesizer.Configuration) + + /// Called when the synthesizer's state is updated. + func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange state: PublicationSpeechSynthesizer.State) + + /// Called when an `error` occurs while speaking `utterance`. + func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, utterance: PublicationSpeechSynthesizer.Utterance, didFailWithError error: PublicationSpeechSynthesizer.Error) +} + +/// `PublicationSpeechSynthesizer` orchestrates the rendition of a `Publication` by iterating through its content, +/// splitting it into individual utterances using a `ContentTokenizer`, then using a `TTSEngine` to read them aloud. +/// +/// Don't forget to call `close()` when you are done using the `PublicationSpeechSynthesizer`. +public class PublicationSpeechSynthesizer: Loggable { + public typealias EngineFactory = () -> TTSEngine + public typealias TokenizerFactory = (_ defaultLanguage: Language?) -> ContentTokenizer + + /// Returns whether the `publication` can be played with a `PublicationSpeechSynthesizer`. + static func canSpeak(publication: Publication) -> Bool { + publication.content() != nil + } + + public enum Error: Swift.Error { + /// Underlying `TTSEngine` error. + case engine(TTSError) + } + + /// User configuration for the text-to-speech engine. + public struct Configuration: Equatable { + /// Language overriding the publication one. + public var defaultLanguage: Language? + /// Identifier for the voice used to speak the utterances. + public var voiceIdentifier: String? + /// Multiplier for the voice speech rate. Normal is 1.0. See `rateMultiplierRange` for the range of values + /// supported by the `TTSEngine`. + public var rateMultiplier: Double + // FIXME: + public var pitch: Double + + public init(defaultLanguage: Language? = nil, voiceIdentifier: String? = nil, rateMultiplier: Double = 1.0, pitch: Double = 1.0) { + self.defaultLanguage = defaultLanguage + self.voiceIdentifier = voiceIdentifier + self.rateMultiplier = rateMultiplier + self.pitch = pitch + } + } + + /// An utterance is an arbitrary text (e.g. sentence) extracted from the publication, that can be synthesized by + /// the TTS engine. + public struct Utterance { + /// Text to be spoken. + public let text: String + /// Locator to the utterance in the publication. + public let locator: Locator + /// Language of this utterance, if it dffers from the default publication language. + public let language: Language? + } + + /// Represents a state of the `PublicationSpeechSynthesizer`. + public enum State { + /// The synthesizer is completely stopped and must be (re)started from a given locator. + case stopped + + /// The synthesizer is paused at the given utterance. + case paused(Utterance) + + /// The TTS engine is synthesizing the associated utterance. + /// `range` will be regularly updated while the utterance is being played. + case playing(Utterance, range: Locator?) + } + + /// Current state of the `PublicationSpeechSynthesizer`. + public private(set) var state: State = .stopped { + didSet { + delegate?.publicationSpeechSynthesizer(self, stateDidChange: state) + } + } + + /// Current configuration of the `PublicationSpeechSynthesizer`. + /// + /// Changes are not immediate, they will be applied for the next utterance. + public var config: Configuration { + didSet { + delegate?.publicationSpeechSynthesizer(self, configDidChange: config) + } + } + + public weak var delegate: PublicationSpeechSynthesizerDelegate? + + private let publication: Publication + private let engineFactory: EngineFactory + private let tokenizerFactory: TokenizerFactory + + /// Creates a `PublicationSpeechSynthesizer` using the given `TTSEngine` factory. + /// + /// Returns null if the publication cannot be synthesized. + /// + /// - Parameters: + /// - publication: Publication which will be iterated through and synthesized. + /// - config: Initial TTS configuration. + /// - engineFactory: Factory to create an instance of `TtsEngine`. Defaults to `AVTTSEngine`. + /// - tokenizerFactory: Factory to create a `ContentTokenizer` which will be used to + /// split each `ContentElement` item into smaller chunks. Splits by sentences by default. + /// - delegate: Optional delegate. + public init?( + publication: Publication, + config: Configuration = Configuration(), + engineFactory: @escaping EngineFactory = { AVTTSEngine() }, + tokenizerFactory: @escaping TokenizerFactory = defaultTokenizerFactory, + delegate: PublicationSpeechSynthesizerDelegate? = nil + ) { + guard Self.canSpeak(publication: publication) else { + return nil + } + + self.publication = publication + self.config = config + self.engineFactory = engineFactory + self.tokenizerFactory = tokenizerFactory + self.delegate = delegate + } + + /// The default content tokenizer will split the `Content.Element` items into individual sentences. + public static let defaultTokenizerFactory: TokenizerFactory = { defaultLanguage in + makeTextContentTokenizer( + defaultLanguage: defaultLanguage, + contextSnippetLength: 50, + textTokenizerFactory: { language in + makeDefaultTextTokenizer(unit: .sentence, language: language) + } + ) + } + + /// Interrupts the `TtsEngine` and closes this `PublicationSpeechSynthesizer`. + public func close() { + if isEngineInitialized { + engine.close() + } + } + + private var isEngineInitialized = false + + private lazy var engine: TTSEngine = { + isEngineInitialized = true + return engineFactory() + }() + + /// Range for the speech rate multiplier. Normal is 1.0. + public var rateMultiplierRange: ClosedRange { + engine.rateMultiplierRange + } + + /// List of synthesizer voices supported by the TTS engine. + public var availableVoices: [TTSVoice] { + engine.availableVoices + } + + /// Returns the first voice with the given `identifier` supported by the TTS `engine`. + /// + /// This can be used to restore the user selected voice after storing it in the user defaults. + func voiceWithIdentifier(_ identifier: String) -> TTSVoice? { + let voice = lastUsedVoice.takeIf { $0.identifier == identifier } + ?? engine.voiceWithIdentifier(identifier) + + lastUsedVoice = voice + return voice + } + + /// Cache for the last requested voice, for performance. + private var lastUsedVoice: TTSVoice? = nil + + /// (Re)starts the synthesizer from the given locator or the beginning of the publication. + func start(from startLocator: Locator? = nil) { + publicationIterator = publication.content(from: startLocator)?.makeIterator() + playNextUtterance(.forward) + } + + /// `Content.Iterator` used to iterate through the `publication`. + private var publicationIterator: ContentIterator? = nil { + didSet { + utterances = CursorList() + } + } + + /// Utterances for the current publication `ContentElement` item. + private var utterances: CursorList = CursorList() + + /// Plays the next utterance in the given `direction`. + private func playNextUtterance(_ direction: Direction) { + guard let utterance = nextUtterance(direction) else { + state = .stopped + return + } + play(utterance) + } + + /// Plays the given `utterance` with the TTS `engine`. + private func play(_ utterance: Utterance) { + state = .playing(utterance, range: nil) + + engine.speak( + TTSUtterance( + text: utterance.text, + delay: 0, + rateMultiplier: config.rateMultiplier, + pitch: config.pitch, + voiceOrLanguage: voiceOrLanguage(for: utterance) + ), + onSpeakRange: { [unowned self] range in + state = .playing( + utterance, + range: utterance.locator.copy( + text: { $0 = utterance.locator.text[range] } + ) + ) + }, + completion: { [unowned self] result in + switch result { + case .success: + playNextUtterance(.forward) + case .failure(let error): + state = .paused(utterance) + delegate?.publicationSpeechSynthesizer(self, utterance: utterance, didFailWithError: .engine(error)) + } + } + ) + } + + /// Returns the user selected voice if it's compatible with the utterance language. Otherwise, falls back on + /// the languages. + private func voiceOrLanguage(for utterance: Utterance) -> Either { + if let voice = config.voiceIdentifier + .flatMap({ id in self.voiceWithIdentifier(id) }) + .takeIf({ voice in utterance.language == nil || utterance.language?.removingRegion() == voice.language.removingRegion() }) + { + return .left(voice) + } else { + return .right(utterance.language + ?? config.defaultLanguage + ?? publication.metadata.language + ?? Language.current + ) + } + } + + /// Gets the next utterance in the given `direction`, or null when reaching the beginning or the end. + private func nextUtterance(_ direction: Direction) -> Utterance? { + guard let utterance = utterances.next(direction) else { + if loadNextUtterances(direction) { + return nextUtterance(direction) + } + return nil + } + return utterance + } + + /// Loads the utterances for the next publication `ContentElement` item in the given `direction`. + private func loadNextUtterances(_ direction: Direction) -> Bool { + do { + guard let content = try publicationIterator?.next(direction) else { + return false + } + + let nextUtterances = try tokenize(content) + .flatMap { utterances(for: $0) } + + if nextUtterances.isEmpty { + return loadNextUtterances(direction) + } + + utterances = CursorList( + list: nextUtterances, + startIndex: { + switch direction { + case .forward: return 0 + case .backward: return nextUtterances.count - 1 + } + }() + ) + + return true + + } catch { + log(.error, error) + return false + } + } + + /// Splits a publication `ContentElement` item into smaller chunks using the provided tokenizer. + /// + /// This is used to split a paragraph into sentences, for example. + func tokenize(_ element: ContentElement) throws -> [ContentElement] { + let tokenizer = tokenizerFactory(config.defaultLanguage ?? publication.metadata.language) + return try tokenizer(element) + } + + /// Splits a publication `ContentElement` item into the utterances to be spoken. + private func utterances(for element: ContentElement) -> [Utterance] { + func utterance(text: String, locator: Locator, language: Language? = nil) -> Utterance? { + guard text.contains(where: { $0.isLetter || $0.isNumber }) else { + return nil + } + + return Utterance( + text: text, + locator: locator, + language: language + // If the language is the same as the one declared globally in the publication, + // we omit it. This way, the app can customize the default language used in the + // configuration. + .takeIf { $0 != publication.metadata.language } + ) + } + + switch element { + case let element as TextContentElement: + return element.segments + .compactMap { segment in + utterance(text: segment.text, locator: segment.locator, language: segment.language) + } + + case let element as TextualContentElement: + guard let text = element.text.takeIf({ !$0.isEmpty }) else { + return [] + } + return Array(ofNotNil: utterance(text: text, locator: element.locator)) + + default: + return [] + } + } +} + +private enum Direction { + case forward, backward +} + +private extension CursorList { + mutating func next(_ direction: Direction) -> Element? { + switch direction { + case .forward: + return next() + case .backward: + return previous() + } + } +} + +private extension ContentIterator { + func next(_ direction: Direction) throws -> ContentElement? { + switch direction { + case .forward: + return try next() + case .backward: + return try previous() + } + } +} diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index cdd53f0bc..2bf97047a 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -29,6 +29,9 @@ public protocol TTSEngine: AnyObject { onSpeakRange: @escaping (Range) -> Void, completion: @escaping (Result) -> Void ) -> Cancellable + + /// Terminates the TTS engine. + func close() } public extension TTSEngine { @@ -51,13 +54,13 @@ public struct TTSUtterance { public let text: String /// Delay before speaking the utterance, in seconds. - public var delay: TimeInterval + public let delay: TimeInterval /// Multiplier for the speech rate. public let rateMultiplier: Double /// Voice pitch. - public var pitch: Double + public let pitch: Double /// Either an explicit voice or the language of the text. If a language is provided, the default voice for this /// language will be used. diff --git a/Sources/Navigator/Toolkit/CursorList.swift b/Sources/Navigator/Toolkit/CursorList.swift new file mode 100644 index 000000000..09c94f494 --- /dev/null +++ b/Sources/Navigator/Toolkit/CursorList.swift @@ -0,0 +1,43 @@ +// +// Copyright 2022 Readium Foundation. All rights reserved. +// Use of this source code is governed by the BSD-style license +// available in the top-level LICENSE file of the project. +// + +import Foundation + +/// A `List` with a mutable cursor index. +struct CursorList { + private let list: [Element] + private let startIndex: Int + + init(list: [Element] = [], startIndex: Int = 0) { + self.list = list + self.startIndex = startIndex + } + + private var index: Int? = nil + + /// Returns the current element. + mutating func current() -> Element? { + moveAndGet(index ?? startIndex) + } + + /// Moves the cursor backward and returns the element, or null when reaching the beginning. + mutating func previous() -> Element? { + moveAndGet(index.map { $0 - 1 } ?? startIndex) + } + + /// Moves the cursor forward and returns the element, or null when reaching the end. + mutating func next() -> Element? { + moveAndGet(index.map { $0 + 1 } ?? startIndex) + } + + private mutating func moveAndGet(_ index: Int) -> Element? { + guard list.indices.contains(index) else { + return nil + } + self.index = index + return list[index] + } +} \ No newline at end of file From 520270724126345625a5172de1bbb22eb550ddf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 8 Aug 2022 17:03:21 +0200 Subject: [PATCH 36/46] Refactor the `TTSViewModel` --- .../TTS/PublicationSpeechSynthesizer.swift | 58 ++++-- .../Services/Content/Content.swift | 2 +- .../Reader/Common/ReaderViewController.swift | 4 +- .../Sources/Reader/Common/TTS/TTSView.swift | 47 +++-- .../Reader/Common/TTS/TTSViewModel.swift | 184 +++++++++++------- 5 files changed, 188 insertions(+), 107 deletions(-) diff --git a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift index f997ed16b..01679b05f 100644 --- a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift +++ b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift @@ -8,9 +8,6 @@ import Foundation import R2Shared public protocol PublicationSpeechSynthesizerDelegate: AnyObject { - /// Called when the synthesizer's configuration is updated. - func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, configDidChange config: PublicationSpeechSynthesizer.Configuration) - /// Called when the synthesizer's state is updated. func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange state: PublicationSpeechSynthesizer.State) @@ -27,7 +24,7 @@ public class PublicationSpeechSynthesizer: Loggable { public typealias TokenizerFactory = (_ defaultLanguage: Language?) -> ContentTokenizer /// Returns whether the `publication` can be played with a `PublicationSpeechSynthesizer`. - static func canSpeak(publication: Publication) -> Bool { + public static func canSpeak(publication: Publication) -> Bool { publication.content() != nil } @@ -90,11 +87,7 @@ public class PublicationSpeechSynthesizer: Loggable { /// Current configuration of the `PublicationSpeechSynthesizer`. /// /// Changes are not immediate, they will be applied for the next utterance. - public var config: Configuration { - didSet { - delegate?.publicationSpeechSynthesizer(self, configDidChange: config) - } - } + public var config: Configuration public weak var delegate: PublicationSpeechSynthesizerDelegate? @@ -169,7 +162,7 @@ public class PublicationSpeechSynthesizer: Loggable { /// Returns the first voice with the given `identifier` supported by the TTS `engine`. /// /// This can be used to restore the user selected voice after storing it in the user defaults. - func voiceWithIdentifier(_ identifier: String) -> TTSVoice? { + public func voiceWithIdentifier(_ identifier: String) -> TTSVoice? { let voice = lastUsedVoice.takeIf { $0.identifier == identifier } ?? engine.voiceWithIdentifier(identifier) @@ -181,11 +174,54 @@ public class PublicationSpeechSynthesizer: Loggable { private var lastUsedVoice: TTSVoice? = nil /// (Re)starts the synthesizer from the given locator or the beginning of the publication. - func start(from startLocator: Locator? = nil) { + public func start(from startLocator: Locator? = nil) { publicationIterator = publication.content(from: startLocator)?.makeIterator() playNextUtterance(.forward) } + /// Stops the synthesizer. + /// + /// Use `start()` to restart it. + public func stop() { + state = .stopped + publicationIterator = nil + } + + /// Interrupts a played utterance. + /// + /// Use `resume()` to restart the playback from the same utterance. + public func pause() { + if case let .playing(utterance, range: _) = state { + state = .paused(utterance) + } + } + + /// Resumes an utterance interrupted with `pause()`. + public func resume() { + if case let .paused(utterance) = state { + play(utterance) + } + } + + /// Pauses or resumes the playback of the current utterance. + public func pauseOrResume() { + switch state { + case .stopped: return + case .playing: pause() + case .paused: resume() + } + } + + /// Skips to the previous utterance. + public func previous() { + playNextUtterance(.backward) + } + + /// Skips to the next utterance. + public func next() { + playNextUtterance(.forward) + } + /// `Content.Iterator` used to iterate through the `publication`. private var publicationIterator: ContentIterator? = nil { didSet { diff --git a/Sources/Shared/Publication/Services/Content/Content.swift b/Sources/Shared/Publication/Services/Content/Content.swift index e1ed7297d..2d5bb0f65 100644 --- a/Sources/Shared/Publication/Services/Content/Content.swift +++ b/Sources/Shared/Publication/Services/Content/Content.swift @@ -79,7 +79,7 @@ public struct ImageContentElement: EmbeddedContentElement, TextualContentElement public var text: String? { // The caption might be a better text description than the accessibility label, when available. - caption.takeIf { !$0.isEmpty } ?? (self as? TextualContentElement)?.text + caption.takeIf { !$0.isEmpty } ?? accessibilityLabel } } diff --git a/TestApp/Sources/Reader/Common/ReaderViewController.swift b/TestApp/Sources/Reader/Common/ReaderViewController.swift index 851456d4f..7cee5530e 100644 --- a/TestApp/Sources/Reader/Common/ReaderViewController.swift +++ b/TestApp/Sources/Reader/Common/ReaderViewController.swift @@ -125,7 +125,7 @@ class ReaderViewController: UIViewController, Loggable { state .sink { state in - controls.view.isHidden = (state == .stopped) + controls.view.isHidden = !state.showControls } .store(in: &subscriptions) } @@ -162,7 +162,7 @@ class ReaderViewController: UIViewController, Loggable { } // Text to speech if let ttsViewModel = ttsViewModel { - buttons.append(UIBarButtonItem(image: UIImage(systemName: "speaker.wave.2.fill"), style: .plain, target: ttsViewModel, action: #selector(TTSViewModel.play))) + buttons.append(UIBarButtonItem(image: UIImage(systemName: "speaker.wave.2.fill"), style: .plain, target: ttsViewModel, action: #selector(TTSViewModel.start))) } return buttons diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index 69790bf78..13349dee0 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -8,6 +8,8 @@ import Foundation import R2Navigator import SwiftUI +private typealias Config = PublicationSpeechSynthesizer.Configuration + struct TTSControls: View { @ObservedObject var viewModel: TTSViewModel @State private var showSettings = false @@ -25,8 +27,8 @@ struct TTSControls: View { ) IconButton( - systemName: (viewModel.state == .playing) ? "pause.fill" : "play.fill", - action: { viewModel.playPause() } + systemName: (viewModel.state.isPlaying) ? "pause.fill" : "play.fill", + action: { viewModel.pauseOrResume() } ) IconButton( @@ -69,32 +71,35 @@ struct TTSSettings: View { @ObservedObject var viewModel: TTSViewModel var body: some View { + let settings = viewModel.settings NavigationView { Form { stepper( caption: "Rate", - for: \.rate, - step: viewModel.defaultConfig.rate / 10 + for: \.rateMultiplier, + step: 0.2 ) stepper( caption: "Pitch", for: \.pitch, - step: viewModel.defaultConfig.pitch / 4 + step: 0.2 ) picker( caption: "Language", for: \.defaultLanguage, - choices: viewModel.availableLanguages, - choiceLabel: { $0.localizedDescription() } + choices: settings.availableLanguages, + choiceLabel: { $0?.localizedDescription() ?? "Default" } ) picker( caption: "Voice", - for: \.voice, - choices: viewModel.availableVoices, - choiceLabel: { $0.localizedDescription() } + for: \.voiceIdentifier, + choices: [nil] + settings.availableVoiceIds, + choiceLabel: { id in + id.flatMap { viewModel.voiceWithIdentifier($0)?.name } ?? "Default" + } ) } .navigationTitle("Speech settings") @@ -103,9 +108,9 @@ struct TTSSettings: View { .navigationViewStyle(.stack) } - @ViewBuilder func stepper( + @ViewBuilder private func stepper( caption: String, - for keyPath: WritableKeyPath, + for keyPath: WritableKeyPath, step: Double ) -> some View { Stepper( @@ -114,13 +119,13 @@ struct TTSSettings: View { step: step ) { Text(caption) - Text(String.localizedPercentage(viewModel.config[keyPath: keyPath])).font(.footnote) + Text(String.localizedPercentage(viewModel.settings.config[keyPath: keyPath])).font(.footnote) } } - @ViewBuilder func picker( + @ViewBuilder private func picker( caption: String, - for keyPath: WritableKeyPath, + for keyPath: WritableKeyPath, choices: [T], choiceLabel: @escaping (T) -> String ) -> some View { @@ -131,10 +136,14 @@ struct TTSSettings: View { } } - private func configBinding(for keyPath: WritableKeyPath) -> Binding { + private func configBinding(for keyPath: WritableKeyPath) -> Binding { Binding( - get: { viewModel.config[keyPath: keyPath] }, - set: { viewModel.config[keyPath: keyPath] = $0 } + get: { viewModel.settings.config[keyPath: keyPath] }, + set: { + var config = viewModel.settings.config + config[keyPath: keyPath] = $0 + viewModel.setConfig(config) + } ) } } @@ -144,7 +153,7 @@ private extension Optional where Wrapped == TTSVoice { guard case let .some(voice) = self else { return "Default" } - var desc = voice.name + var desc = voice.name ?? "Voice" if let region = voice.language.localizedRegion() { desc += " (\(region))" } diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index d03ca95ae..c6a54b03a 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -11,44 +11,86 @@ import R2Shared final class TTSViewModel: ObservableObject, Loggable { - enum State { - case stopped, paused, playing + struct State: Equatable { + /// Whether the TTS was enabled by the user. + var showControls: Bool = false + /// Whether the TTS is currently speaking. + var isPlaying: Bool = false } - @Published private(set) var state: State = .stopped - @Published var config: TTSConfiguration + struct Settings: Equatable { + /// Currently selected user preferences. + let config: PublicationSpeechSynthesizer.Configuration + /// Languages supported by the synthesizer. + let availableLanguages: [Language] + /// Voices supported by the synthesizer, for the selected language. + let availableVoiceIds: [String] + /// Supported range for the rate multiplier. + let rateMultiplierRange: ClosedRange + + init(synthesizer: PublicationSpeechSynthesizer) { + let voicesByLanguage: [Language: [TTSVoice]] = + Dictionary(grouping: synthesizer.availableVoices, by: \.language) + + self.config = synthesizer.config + self.availableLanguages = voicesByLanguage.keys.sorted { $0.localizedDescription() < $1.localizedDescription() } + self.availableVoiceIds = synthesizer.config.defaultLanguage + .flatMap { voicesByLanguage[$0]?.map { $0.identifier } } + ?? [] + self.rateMultiplierRange = synthesizer.rateMultiplierRange + } + } + + @Published private(set) var state: State = State() + @Published private(set) var settings: Settings - private let tts: TTSController - private let navigator: Navigator private let publication: Publication + private let navigator: Navigator + private let synthesizer: PublicationSpeechSynthesizer + + @Published private var playingUtterance: Locator? + private let playingWordRangeSubject = PassthroughSubject() - private let playingRangeLocatorSubject = PassthroughSubject() private var subscriptions: Set = [] init?(navigator: Navigator, publication: Publication) { - guard TTSController.canPlay(publication) else { + guard let synthesizer = PublicationSpeechSynthesizer(publication: publication) else { return nil } - self.tts = TTSController(publication: publication) - self.config = tts.config + self.synthesizer = synthesizer + self.settings = Settings(synthesizer: synthesizer) self.navigator = navigator self.publication = publication - tts.delegate = self - - $config - .sink { [unowned self] in - tts.config = $0 - } - .store(in: &subscriptions) + synthesizer.delegate = self + + // Highlight the currently spoken utterance. + if let navigator = navigator as? DecorableNavigator { + $playingUtterance + .removeDuplicates() + .sink { locator in + var decorations: [Decoration] = [] + if let locator = locator { + decorations.append(Decoration( + id: "tts-utterance", + locator: locator, + style: .highlight(tint: .red) + )) + } + navigator.apply(decorations: decorations, in: "tts") + } + .store(in: &subscriptions) + } + // Navigate to the currently spoken utterance word. + // This will automatically turn pages when needed. var isMoving = false - playingRangeLocatorSubject + playingWordRangeSubject + .removeDuplicates() + // Improve performances by throttling the moves to maximum one per second. .throttle(for: 1, scheduler: RunLoop.main, latest: true) + .drop(while: { _ in isMoving }) .sink { locator in - guard !isMoving else { - return - } isMoving = navigator.go(to: locator) { isMoving = false } @@ -56,79 +98,73 @@ final class TTSViewModel: ObservableObject, Loggable { .store(in: &subscriptions) } - var defaultConfig: TTSConfiguration { tts.defaultConfig } - - var availableVoices: [TTSVoice?] { - [nil] + tts.availableVoices - .filter { $0.language.removingRegion() == config.defaultLanguage } + func setConfig(_ config: PublicationSpeechSynthesizer.Configuration) { + synthesizer.config = config + settings = Settings(synthesizer: synthesizer) } - lazy var availableLanguages: [Language] = - tts.availableVoices - .map { $0.language.removingRegion() } - .removingDuplicates() - .sorted { $0.localizedDescription() < $1.localizedDescription() } - - @objc func play() { - navigator.firstVisibleElementLocator { [self] locator in - tts.play(from: locator ?? navigator.currentLocation) - } + func voiceWithIdentifier(_ id: String) -> TTSVoice? { + synthesizer.voiceWithIdentifier(id) } - @objc func playPause() { - tts.playPause() + @objc func start() { + if let navigator = navigator as? VisualNavigator { + // Gets the locator of the element at the top of the page. + navigator.firstVisibleElementLocator { [self] locator in + synthesizer.start(from: locator) + } + } else { + synthesizer.start(from: navigator.currentLocation) + } } @objc func stop() { - state = .stopped - highlight(nil) - tts.pause() + synthesizer.stop() } - @objc func previous() { - tts.previous() + @objc func pauseOrResume() { + synthesizer.pauseOrResume() } - @objc func next() { - tts.next() + @objc func pause() { + synthesizer.pause() } - private func highlight(_ utterance: TTSUtterance?) { - guard let navigator = navigator as? DecorableNavigator else { - return - } - - var decorations: [Decoration] = [] - if let utterance = utterance { - decorations.append(Decoration( - id: "tts", - locator: utterance.locator, - style: .highlight(tint: .red) - )) - } + @objc func previous() { + synthesizer.previous() + } - navigator.apply(decorations: decorations, in: "tts") + @objc func next() { + synthesizer.next() } } -extension TTSViewModel: TTSControllerDelegate { - public func ttsController(_ ttsController: TTSController, playingDidChange isPlaying: Bool) { - if isPlaying { - state = .playing - } else if state != .stopped { - state = .paused +extension TTSViewModel: PublicationSpeechSynthesizerDelegate { + + public func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange synthesizerState: PublicationSpeechSynthesizer.State) { + switch synthesizerState { + case .stopped: + state.showControls = false + state.isPlaying = false + playingUtterance = nil + + case let .playing(utterance, range: wordRange): + state.showControls = true + state.isPlaying = true + playingUtterance = utterance.locator + if let wordRange = wordRange { + playingWordRangeSubject.send(wordRange) + } + + case let .paused(utterance): + state.showControls = true + state.isPlaying = false + playingUtterance = utterance.locator } } - public func ttsController(_ ttsController: TTSController, didReceiveError error: Error) { + public func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, utterance: PublicationSpeechSynthesizer.Utterance, didFailWithError error: PublicationSpeechSynthesizer.Error) { + // FIXME: log(.error, error) } - - public func ttsController(_ ttsController: TTSController, willStartSpeaking utterance: TTSUtterance) { - highlight(utterance) - } - - public func ttsController(_ ttsController: TTSController, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) { - playingRangeLocatorSubject.send(locator) - } } From 71ac66b0530f11b3a46628602b116bc07e0c8c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 8 Aug 2022 17:34:06 +0200 Subject: [PATCH 37/46] Auto-cancel tasks --- Sources/Navigator/TTS/AVTTSEngine.swift | 36 ++++++++----------- .../TTS/PublicationSpeechSynthesizer.swift | 22 +++++------- Sources/Navigator/TTS/TTSEngine.swift | 3 -- 3 files changed, 24 insertions(+), 37 deletions(-) diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index f3b1b0dc0..c06deef6c 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -58,10 +58,6 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg synthesizer.delegate = self } - public func close() { - on(.close) - } - // FIXME: Double check public let rateMultiplierRange: ClosedRange = 0.5...2.0 @@ -79,7 +75,6 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg onSpeakRange: @escaping (Range) -> (), completion: @escaping (Result) -> () ) -> Cancellable { - let task = Task( utterance: utterance, onSpeakRange: onSpeakRange, @@ -95,7 +90,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg return cancellable } - private class Task: Equatable { + private class Task: Equatable, CustomStringConvertible { let utterance: TTSUtterance let onSpeakRange: (Range) -> () let completion: (Result) -> () @@ -110,6 +105,10 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg static func ==(lhs: Task, rhs: Task) -> Bool { ObjectIdentifier(lhs) == ObjectIdentifier(rhs) } + + var description: String { + utterance.text + } } private func taskUtterance(with task: Task) -> TaskUtterance { @@ -211,8 +210,6 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// State machine events triggered by the `AVSpeechSynthesizer` or the client /// of `AVTTSEngine`. private enum Event: Equatable { - case close - // AVTTSEngine commands case play(Task) case stop(Task) @@ -234,7 +231,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// Raises a TTS event triggering a state change and handles its side effects. private func on(_ event: Event) { assert(Thread.isMainThread, "Raising AVTTSEngine events must be done from the main thread") - + if (debug) { log(.debug, "-> on \(event)") } @@ -242,9 +239,6 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg switch (state, event) { // stopped - case (_, .close): - stopEngine() - case let (.stopped, .play(task)): state = .starting(task) startEngine(with: task) @@ -257,22 +251,22 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg case let (.starting(current), .play(next)): state = .stopping(current, queued: next) - case let (.starting(current), .stop): + case let (.starting(current), .stop(toStop)) where current == toStop: state = .stopping(current, queued: nil) // playing case let (.playing(current), .didFinish(finished)) where current == finished: - current.completion(.success(())) state = .stopped + current.completion(.success(())) case let (.playing(current), .play(next)): - stopEngine() state = .stopping(current, queued: next) - - case let (.playing(current), .stop): stopEngine() + + case let (.playing(current), .stop(toStop)) where current == toStop: state = .stopping(current, queued: nil) + stopEngine() case let (.playing(current), .willSpeakRange(range, task: speaking)) where current == speaking: current.onSpeakRange(range) @@ -280,22 +274,22 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg // stopping case let (.stopping(current, queued: next), .didStart(started)) where current == started: - stopEngine() state = .stopping(current, queued: next) + stopEngine() case let (.stopping(current, queued: next), .didFinish(finished)) where current == finished: if let next = next { - startEngine(with: next) state = .starting(next) + startEngine(with: next) } else { - current.completion(.success(())) state = .stopped + current.completion(.success(())) } case let (.stopping(current, queued: _), .play(next)): state = .stopping(current, queued: next) - case let (.stopping(current, queued: _), .stop): + case let (.stopping(current, queued: _), .stop(toStop)) where current == toStop: state = .stopping(current, queued: nil) diff --git a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift index 01679b05f..f1fb2b830 100644 --- a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift +++ b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift @@ -135,19 +135,9 @@ public class PublicationSpeechSynthesizer: Loggable { ) } - /// Interrupts the `TtsEngine` and closes this `PublicationSpeechSynthesizer`. - public func close() { - if isEngineInitialized { - engine.close() - } - } - - private var isEngineInitialized = false + private var currentTask: Cancellable? = nil - private lazy var engine: TTSEngine = { - isEngineInitialized = true - return engineFactory() - }() + private lazy var engine: TTSEngine = engineFactory() /// Range for the speech rate multiplier. Normal is 1.0. public var rateMultiplierRange: ClosedRange { @@ -175,6 +165,7 @@ public class PublicationSpeechSynthesizer: Loggable { /// (Re)starts the synthesizer from the given locator or the beginning of the publication. public func start(from startLocator: Locator? = nil) { + currentTask?.cancel() publicationIterator = publication.content(from: startLocator)?.makeIterator() playNextUtterance(.forward) } @@ -183,6 +174,7 @@ public class PublicationSpeechSynthesizer: Loggable { /// /// Use `start()` to restart it. public func stop() { + currentTask?.cancel() state = .stopped publicationIterator = nil } @@ -191,6 +183,7 @@ public class PublicationSpeechSynthesizer: Loggable { /// /// Use `resume()` to restart the playback from the same utterance. public func pause() { + currentTask?.cancel() if case let .playing(utterance, range: _) = state { state = .paused(utterance) } @@ -198,6 +191,7 @@ public class PublicationSpeechSynthesizer: Loggable { /// Resumes an utterance interrupted with `pause()`. public func resume() { + currentTask?.cancel() if case let .paused(utterance) = state { play(utterance) } @@ -214,11 +208,13 @@ public class PublicationSpeechSynthesizer: Loggable { /// Skips to the previous utterance. public func previous() { + currentTask?.cancel() playNextUtterance(.backward) } /// Skips to the next utterance. public func next() { + currentTask?.cancel() playNextUtterance(.forward) } @@ -245,7 +241,7 @@ public class PublicationSpeechSynthesizer: Loggable { private func play(_ utterance: Utterance) { state = .playing(utterance, range: nil) - engine.speak( + currentTask = engine.speak( TTSUtterance( text: utterance.text, delay: 0, diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index 2bf97047a..f410680f7 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -29,9 +29,6 @@ public protocol TTSEngine: AnyObject { onSpeakRange: @escaping (Range) -> Void, completion: @escaping (Result) -> Void ) -> Cancellable - - /// Terminates the TTS engine. - func close() } public extension TTSEngine { From 7a9879b9737bb563645b4be0c3aec850817c5ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 8 Aug 2022 17:52:35 +0200 Subject: [PATCH 38/46] Fix pause --- Sources/Navigator/TTS/AVTTSEngine.swift | 17 +- Sources/Navigator/TTS/TTSController.swift | 294 ---------------------- 2 files changed, 13 insertions(+), 298 deletions(-) delete mode 100644 Sources/Navigator/TTS/TTSController.swift diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index c06deef6c..65131ed3b 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -102,13 +102,17 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg self.completion = completion } - static func ==(lhs: Task, rhs: Task) -> Bool { - ObjectIdentifier(lhs) == ObjectIdentifier(rhs) + var isCancelled: Bool { + cancellable?.isCancelled ?? false } var description: String { utterance.text } + + static func ==(lhs: Task, rhs: Task) -> Bool { + ObjectIdentifier(lhs) == ObjectIdentifier(rhs) + } } private func taskUtterance(with task: Task) -> TaskUtterance { @@ -269,7 +273,9 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg stopEngine() case let (.playing(current), .willSpeakRange(range, task: speaking)) where current == speaking: - current.onSpeakRange(range) + if !current.isCancelled { + current.onSpeakRange(range) + } // stopping @@ -278,11 +284,14 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg stopEngine() case let (.stopping(current, queued: next), .didFinish(finished)) where current == finished: - if let next = next { + if let next = next, !next.isCancelled { state = .starting(next) startEngine(with: next) } else { state = .stopped + } + + if !current.isCancelled { current.completion(.success(())) } diff --git a/Sources/Navigator/TTS/TTSController.swift b/Sources/Navigator/TTS/TTSController.swift deleted file mode 100644 index ac7916786..000000000 --- a/Sources/Navigator/TTS/TTSController.swift +++ /dev/null @@ -1,294 +0,0 @@ -// -// Copyright 2022 Readium Foundation. All rights reserved. -// Use of this source code is governed by the BSD-style license -// available in the top-level LICENSE file of the project. -// - -import Foundation -import R2Shared - -public protocol TTSControllerDelegate: AnyObject { - func ttsController(_ ttsController: TTSController, playingDidChange isPlaying: Bool) - - func ttsController(_ ttsController: TTSController, willStartSpeaking utterance: TTSUtterance) - func ttsController(_ ttsController: TTSController, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) - - func ttsController(_ ttsController: TTSController, didReceiveError error: Error) -} - -public extension TTSControllerDelegate { - func ttsController(_ ttsController: TTSController, playingDidChange isPlaying: Bool) {} - func ttsController(_ ttsController: TTSController, willStartSpeaking utterance: TTSUtterance) {} - func ttsController(_ ttsController: TTSController, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) {} -} - -public struct TTSUtterance2: Equatable { - public let text: String - public let locator: Locator - public let language: Language? - public let delay: TimeInterval -} - -public class TTSController: Loggable, TTSEngineDelegate { - public typealias TokenizerFactory = (_ language: Language?) -> ContentTokenizer - - public static func canPlay(_ publication: Publication) -> Bool { - publication.isContentIterable - } - - public var defaultConfig: TTSConfiguration { - engine.defaultConfig - } - - public var config: TTSConfiguration { - get { engine.config } - set { engine.config = newValue } - } - - public var availableVoices: [TTSVoice] { - engine.availableVoices - } - - public func voiceWithIdentifier(_ id: String) -> TTSVoice? { - engine.voiceWithIdentifier(id) - } - - public weak var delegate: TTSControllerDelegate? - - private let publication: Publication - private let engine: TTSEngine - private let makeTokenizer: TokenizerFactory - private let queue: DispatchQueue = .global(qos: .userInitiated) - - public init( - publication: Publication, - engine: TTSEngine = AVTTSEngine(), - makeTokenizer: TokenizerFactory? = nil, - delegate: TTSControllerDelegate? = nil - ) { - precondition(publication.isContentIterable, "The Publication must be iterable to be used with TTSController") - - self.delegate = delegate - self.publication = publication - self.engine = engine - self.makeTokenizer = { language in - makeTextContentTokenizer( - unit: .sentence, - language: language - ) - } - - if let language = publication.metadata.language { - engine.config.defaultLanguage = language - } - engine.delegate = self - } - - deinit { - engine.stop() - contentIterator?.close() - } - - public var isPlaying: Bool = false { - didSet { - precondition(Thread.isMainThread, "TTSController.isPlaying must be mutated from the main thread") - if oldValue != isPlaying { - delegate?.ttsController(self, playingDidChange: isPlaying) - } - } - } - - public func playPause(from start: Locator? = nil) { - if isPlaying { - pause() - } else { - play(from: start) - } - } - - public func pause() { - precondition(Thread.isMainThread, "TTSController.pause() must be called from the main thread") - isPlaying = false - engine.stop() - } - - public func play(from start: Locator? = nil) { - if start != nil { - speakingUtteranceIndex = nil - utterances = [] - contentIterator = publication.contentIterator(from: start) - } - - if contentIterator == nil { - contentIterator = publication.contentIterator(from: nil) - } - - if let utterance = currentUtterance { - play(utterance) - } else { - next() - } - } - - public func previous() { - playNextUtterance(direction: .backward) - } - - public func next() { - playNextUtterance(direction: .forward) - } - - private enum Direction { - case forward, backward - } - - private var contentIterator: ContentIteratorOld? { - willSet { contentIterator?.close() } - } - - private var speakingUtteranceIndex: Int? - private var utterances: [TTSUtterance] = [] - - private var currentUtterance: TTSUtterance? { - speakingUtteranceIndex.map { utterances[$0] } - } - - private func playNextUtterance(direction: Direction) { - queue.async { [self] in - do { - let utterance = try nextUtterance(direction: direction) - DispatchQueue.main.async { [self] in - if let utterance = utterance { - play(utterance) - } else { - isPlaying = false - } - } - } catch { - DispatchQueue.main.async { [self] in - delegate?.ttsController(self, didReceiveError: error) - } - } - } - } - - private func play(_ utterance: TTSUtterance) { - precondition(Thread.isMainThread, "TTSController.play() must be called from the main thread") - - delegate?.ttsController(self, willStartSpeaking: utterance) - isPlaying = true - engine.speak(utterance) - } - - private func nextUtterance(direction: Direction) throws -> TTSUtterance? { - guard let nextIndex = nextUtteranceIndex(direction: direction) else { - if try loadNextUtterances(direction: direction) { - return try nextUtterance(direction: direction) - } else { - return nil - } - } - speakingUtteranceIndex = nextIndex - return utterances[nextIndex] - } - - private func nextUtteranceIndex(direction: Direction) -> Int? { - let index: Int = { - switch direction { - case .forward: - return (speakingUtteranceIndex ?? -1) + 1 - case .backward: - return (speakingUtteranceIndex ?? utterances.count) - 1 - } - }() - guard utterances.indices.contains(index) else { - return nil - } - return index - } - - private func loadNextUtterances(direction: Direction) throws -> Bool { - speakingUtteranceIndex = nil - utterances = [] - - guard let content: ContentOld = try { - switch direction { - case .forward: - return try contentIterator?.next() - case .backward: - return try contentIterator?.previous() - } - }() else { - return false - } - - utterances = tokenize(content, with: makeTokenizer(nil)) - .flatMap { utterances(from: $0) } - - guard !utterances.isEmpty else { - return try loadNextUtterances(direction: direction) - } - - return true - } - - private func utterances(from content: ContentOld) -> [TTSUtterance] { - switch content.data { - case .audio(target: _): - return [] - - case .image(target: _, description: let description): - guard let description = description, !description.isEmpty else { - return [] - } - return Array(ofNotNil: utterance(text: description, locator: content.locator)) - - case .text(spans: let spans, style: _): - return spans - .enumerated() - .compactMap { offset, span in - utterance( - text: span.text, - locator: span.locator, - language: span.language, - delay: (offset == 0) ? 0.4 : 0 - ) - } - } - } - - private func utterance(text: String, locator: Locator, language: Language? = nil, delay: TimeInterval = 0) -> TTSUtterance? { - guard text.contains(where: { $0.isLetter || $0.isNumber }) else { - return nil - } - return TTSUtterance( - text: text, - locator: locator, - language: language.takeIf { $0 != publication.metadata.language }, - delay: delay - ) - } - - private func tokenize(_ content: ContentOld, with tokenizer: ContentTokenizer) -> [ContentOld] { - do { - return try tokenizer(content) - } catch { - log(.error, error) - return [content] - } - } - - // MARK: - TTSEngineDelegate - - public func ttsEngine(_ engine: TTSEngine, didStopAfterLastUtterance utterance: TTSUtterance) { - if isPlaying { - next() - } - } - - public func ttsEngine(_ engine: TTSEngine, willSpeakRangeAt locator: Locator, of utterance: TTSUtterance) { - DispatchQueue.main.async { [self] in - delegate?.ttsController(self, willSpeakRangeAt: locator, of: utterance) - } - } -} From d182ec7ef1cbefb57564912854f90a3f9fe50cc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Mon, 8 Aug 2022 20:53:17 +0200 Subject: [PATCH 39/46] Fix pitch --- Sources/Navigator/TTS/AVTTSEngine.swift | 9 +++++---- .../TTS/PublicationSpeechSynthesizer.swift | 18 ++++++++++++------ Sources/Navigator/TTS/TTSEngine.swift | 7 +++++-- .../Sources/Reader/Common/TTS/TTSView.swift | 11 +++++++---- .../Reader/Common/TTS/TTSViewModel.swift | 3 +++ 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index 65131ed3b..8bcbae066 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -17,7 +17,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// > `AVSpeechUtteranceMaximumSpeechRate`. Lower values correspond to slower speech, and higher values correspond to /// > faster speech. The default value is `AVSpeechUtteranceDefaultSpeechRate`. /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619708-rate - private let avRateRange = + private static let avRateRange = Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate) /// Range of valid values for an AVUtterance pitch. @@ -25,7 +25,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// > Before enqueuing the utterance, set this property to a value within the range of 0.5 for lower pitch to 2.0 for /// > higher pitch. The default value is 1.0. /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier - private let avPitchRange = 0.5...2.0 + private static let avPitchRange = 0.5...2.0 /// Conversion function between a rate multiplier and the `AVSpeechUtterance` rate. private let rateMultiplierToAVRate: (Double) -> Float @@ -61,6 +61,8 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg // FIXME: Double check public let rateMultiplierRange: ClosedRange = 0.5...2.0 + public let pitchMultiplierRange: ClosedRange = avPitchRange + public lazy var availableVoices: [TTSVoice] = AVSpeechSynthesisVoice.speechVoices() .map { TTSVoice(voice: $0) } @@ -118,8 +120,7 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg private func taskUtterance(with task: Task) -> TaskUtterance { let utter = TaskUtterance(task: task) utter.rate = rateMultiplierToAVRate(task.utterance.rateMultiplier) - // FIXME: -// utter.pitchMultiplier = Float(avPitchRange.valueForPercentage(config.pitch)) + utter.pitchMultiplier = Float(task.utterance.pitchMultiplier) utter.preUtteranceDelay = task.utterance.delay utter.voice = voice(for: task.utterance) return utter diff --git a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift index f1fb2b830..24ebc4921 100644 --- a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift +++ b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift @@ -39,17 +39,18 @@ public class PublicationSpeechSynthesizer: Loggable { public var defaultLanguage: Language? /// Identifier for the voice used to speak the utterances. public var voiceIdentifier: String? - /// Multiplier for the voice speech rate. Normal is 1.0. See `rateMultiplierRange` for the range of values + /// Multiplier for the voice speech rate. Normal is 1.0, see `rateMultiplierRange` for the range of values /// supported by the `TTSEngine`. public var rateMultiplier: Double - // FIXME: - public var pitch: Double + /// Multiplier for the baseline voice pitch. Normal is 1.0, see `pichMultiplierRange` for the range of values + /// supported by the `TTSEngine`. + public var pitchMultiplier: Double - public init(defaultLanguage: Language? = nil, voiceIdentifier: String? = nil, rateMultiplier: Double = 1.0, pitch: Double = 1.0) { + public init(defaultLanguage: Language? = nil, voiceIdentifier: String? = nil, rateMultiplier: Double = 1.0, pitchMultiplier: Double = 1.0) { self.defaultLanguage = defaultLanguage self.voiceIdentifier = voiceIdentifier self.rateMultiplier = rateMultiplier - self.pitch = pitch + self.pitchMultiplier = pitchMultiplier } } @@ -144,6 +145,11 @@ public class PublicationSpeechSynthesizer: Loggable { engine.rateMultiplierRange } + /// Range for the voice pitch multiplier. Normal is 1.0. + public var pitchMultiplierRange: ClosedRange { + engine.pitchMultiplierRange + } + /// List of synthesizer voices supported by the TTS engine. public var availableVoices: [TTSVoice] { engine.availableVoices @@ -246,7 +252,7 @@ public class PublicationSpeechSynthesizer: Loggable { text: utterance.text, delay: 0, rateMultiplier: config.rateMultiplier, - pitch: config.pitch, + pitchMultiplier: config.pitchMultiplier, voiceOrLanguage: voiceOrLanguage(for: utterance) ), onSpeakRange: { [unowned self] range in diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index f410680f7..5c11e36db 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -15,6 +15,9 @@ public protocol TTSEngine: AnyObject { /// Supported range for the speech rate multiplier. var rateMultiplierRange: ClosedRange { get } + /// Supported range for the voice pitch multiplier. + var pitchMultiplierRange: ClosedRange { get } + /// List of available synthesizer voices. var availableVoices: [TTSVoice] { get } @@ -56,8 +59,8 @@ public struct TTSUtterance { /// Multiplier for the speech rate. public let rateMultiplier: Double - /// Voice pitch. - public let pitch: Double + /// Multiplier for the baseline voice pitch. + public let pitchMultiplier: Double /// Either an explicit voice or the language of the text. If a language is provided, the default voice for this /// language will be used. diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index 13349dee0..f5d31de58 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -77,13 +77,15 @@ struct TTSSettings: View { stepper( caption: "Rate", for: \.rateMultiplier, - step: 0.2 + range: settings.rateMultiplierRange, + step: 0.1 ) stepper( caption: "Pitch", - for: \.pitch, - step: 0.2 + for: \.pitchMultiplier, + range: settings.pitchMultiplierRange, + step: 0.1 ) picker( @@ -111,11 +113,12 @@ struct TTSSettings: View { @ViewBuilder private func stepper( caption: String, for keyPath: WritableKeyPath, + range: ClosedRange, step: Double ) -> some View { Stepper( value: configBinding(for: keyPath), - in: 0.0...1.0, + in: range, step: step ) { Text(caption) diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index c6a54b03a..36087f8b6 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -27,6 +27,8 @@ final class TTSViewModel: ObservableObject, Loggable { let availableVoiceIds: [String] /// Supported range for the rate multiplier. let rateMultiplierRange: ClosedRange + /// Supported range for the voice pitch multiplier. + let pitchMultiplierRange: ClosedRange init(synthesizer: PublicationSpeechSynthesizer) { let voicesByLanguage: [Language: [TTSVoice]] = @@ -38,6 +40,7 @@ final class TTSViewModel: ObservableObject, Loggable { .flatMap { voicesByLanguage[$0]?.map { $0.identifier } } ?? [] self.rateMultiplierRange = synthesizer.rateMultiplierRange + self.pitchMultiplierRange = synthesizer.pitchMultiplierRange } } From eb597abbb8d184bc3d510ec9bad1af2837ddecdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sat, 13 Aug 2022 10:10:54 +0200 Subject: [PATCH 40/46] Remove rate and pitch settings for now (will be available with the Settings API) --- Sources/Navigator/TTS/AVTTSEngine.swift | 23 +++----------- .../TTS/PublicationSpeechSynthesizer.swift | 24 ++------------- Sources/Navigator/TTS/TTSEngine.swift | 12 -------- .../Sources/Reader/Common/TTS/TTSView.swift | 30 ------------------- .../Reader/Common/TTS/TTSViewModel.swift | 6 ---- 5 files changed, 6 insertions(+), 89 deletions(-) diff --git a/Sources/Navigator/TTS/AVTTSEngine.swift b/Sources/Navigator/TTS/AVTTSEngine.swift index 8bcbae066..125934e4d 100644 --- a/Sources/Navigator/TTS/AVTTSEngine.swift +++ b/Sources/Navigator/TTS/AVTTSEngine.swift @@ -27,42 +27,27 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg /// > https://developer.apple.com/documentation/avfaudio/avspeechutterance/1619683-pitchmultiplier private static let avPitchRange = 0.5...2.0 - /// Conversion function between a rate multiplier and the `AVSpeechUtterance` rate. - private let rateMultiplierToAVRate: (Double) -> Float - - private let debug: Bool + private let debug: Bool = false private let synthesizer = AVSpeechSynthesizer() /// Creates a new `AVTTSEngine` instance. /// /// - Parameters: - /// - rateMultiplierToAVRate: Conversion function between a rate multiplier and the `AVSpeechUtterance` rate. /// - audioSessionConfig: AudioSession configuration used while playing utterances. If `nil`, utterances won't /// play when the app is in the background. - /// - debug: Print the state machine transitions. public init( - // FIXME: - rateMultiplierToAVRate: @escaping (Double) -> Float = { Float($0) }, audioSessionConfig: _AudioSession.Configuration? = .init( category: .playback, mode: .spokenAudio, options: .mixWithOthers - ), - debug: Bool = false + ) ) { - self.rateMultiplierToAVRate = rateMultiplierToAVRate self.audioSessionUser = audioSessionConfig.map { AudioSessionUser(config: $0) } - self.debug = debug super.init() synthesizer.delegate = self } - // FIXME: Double check - public let rateMultiplierRange: ClosedRange = 0.5...2.0 - - public let pitchMultiplierRange: ClosedRange = avPitchRange - public lazy var availableVoices: [TTSVoice] = AVSpeechSynthesisVoice.speechVoices() .map { TTSVoice(voice: $0) } @@ -119,8 +104,8 @@ public class AVTTSEngine: NSObject, TTSEngine, AVSpeechSynthesizerDelegate, Logg private func taskUtterance(with task: Task) -> TaskUtterance { let utter = TaskUtterance(task: task) - utter.rate = rateMultiplierToAVRate(task.utterance.rateMultiplier) - utter.pitchMultiplier = Float(task.utterance.pitchMultiplier) +// utter.rate = rateMultiplierToAVRate(task.utterance.rateMultiplier) +// utter.pitchMultiplier = Float(task.utterance.pitchMultiplier) utter.preUtteranceDelay = task.utterance.delay utter.voice = voice(for: task.utterance) return utter diff --git a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift index 24ebc4921..5b37941e6 100644 --- a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift +++ b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift @@ -39,18 +39,10 @@ public class PublicationSpeechSynthesizer: Loggable { public var defaultLanguage: Language? /// Identifier for the voice used to speak the utterances. public var voiceIdentifier: String? - /// Multiplier for the voice speech rate. Normal is 1.0, see `rateMultiplierRange` for the range of values - /// supported by the `TTSEngine`. - public var rateMultiplier: Double - /// Multiplier for the baseline voice pitch. Normal is 1.0, see `pichMultiplierRange` for the range of values - /// supported by the `TTSEngine`. - public var pitchMultiplier: Double - - public init(defaultLanguage: Language? = nil, voiceIdentifier: String? = nil, rateMultiplier: Double = 1.0, pitchMultiplier: Double = 1.0) { + + public init(defaultLanguage: Language? = nil, voiceIdentifier: String? = nil) { self.defaultLanguage = defaultLanguage self.voiceIdentifier = voiceIdentifier - self.rateMultiplier = rateMultiplier - self.pitchMultiplier = pitchMultiplier } } @@ -140,16 +132,6 @@ public class PublicationSpeechSynthesizer: Loggable { private lazy var engine: TTSEngine = engineFactory() - /// Range for the speech rate multiplier. Normal is 1.0. - public var rateMultiplierRange: ClosedRange { - engine.rateMultiplierRange - } - - /// Range for the voice pitch multiplier. Normal is 1.0. - public var pitchMultiplierRange: ClosedRange { - engine.pitchMultiplierRange - } - /// List of synthesizer voices supported by the TTS engine. public var availableVoices: [TTSVoice] { engine.availableVoices @@ -251,8 +233,6 @@ public class PublicationSpeechSynthesizer: Loggable { TTSUtterance( text: utterance.text, delay: 0, - rateMultiplier: config.rateMultiplier, - pitchMultiplier: config.pitchMultiplier, voiceOrLanguage: voiceOrLanguage(for: utterance) ), onSpeakRange: { [unowned self] range in diff --git a/Sources/Navigator/TTS/TTSEngine.swift b/Sources/Navigator/TTS/TTSEngine.swift index 5c11e36db..8c4db3386 100644 --- a/Sources/Navigator/TTS/TTSEngine.swift +++ b/Sources/Navigator/TTS/TTSEngine.swift @@ -12,12 +12,6 @@ import R2Shared /// Implement this interface to support third-party engines with `PublicationSpeechSynthesizer`. public protocol TTSEngine: AnyObject { - /// Supported range for the speech rate multiplier. - var rateMultiplierRange: ClosedRange { get } - - /// Supported range for the voice pitch multiplier. - var pitchMultiplierRange: ClosedRange { get } - /// List of available synthesizer voices. var availableVoices: [TTSVoice] { get } @@ -56,12 +50,6 @@ public struct TTSUtterance { /// Delay before speaking the utterance, in seconds. public let delay: TimeInterval - /// Multiplier for the speech rate. - public let rateMultiplier: Double - - /// Multiplier for the baseline voice pitch. - public let pitchMultiplier: Double - /// Either an explicit voice or the language of the text. If a language is provided, the default voice for this /// language will be used. public let voiceOrLanguage: Either diff --git a/TestApp/Sources/Reader/Common/TTS/TTSView.swift b/TestApp/Sources/Reader/Common/TTS/TTSView.swift index f5d31de58..9062cdf70 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSView.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSView.swift @@ -74,20 +74,6 @@ struct TTSSettings: View { let settings = viewModel.settings NavigationView { Form { - stepper( - caption: "Rate", - for: \.rateMultiplier, - range: settings.rateMultiplierRange, - step: 0.1 - ) - - stepper( - caption: "Pitch", - for: \.pitchMultiplier, - range: settings.pitchMultiplierRange, - step: 0.1 - ) - picker( caption: "Language", for: \.defaultLanguage, @@ -110,22 +96,6 @@ struct TTSSettings: View { .navigationViewStyle(.stack) } - @ViewBuilder private func stepper( - caption: String, - for keyPath: WritableKeyPath, - range: ClosedRange, - step: Double - ) -> some View { - Stepper( - value: configBinding(for: keyPath), - in: range, - step: step - ) { - Text(caption) - Text(String.localizedPercentage(viewModel.settings.config[keyPath: keyPath])).font(.footnote) - } - } - @ViewBuilder private func picker( caption: String, for keyPath: WritableKeyPath, diff --git a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift index 36087f8b6..e20ec4cc8 100644 --- a/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift +++ b/TestApp/Sources/Reader/Common/TTS/TTSViewModel.swift @@ -25,10 +25,6 @@ final class TTSViewModel: ObservableObject, Loggable { let availableLanguages: [Language] /// Voices supported by the synthesizer, for the selected language. let availableVoiceIds: [String] - /// Supported range for the rate multiplier. - let rateMultiplierRange: ClosedRange - /// Supported range for the voice pitch multiplier. - let pitchMultiplierRange: ClosedRange init(synthesizer: PublicationSpeechSynthesizer) { let voicesByLanguage: [Language: [TTSVoice]] = @@ -39,8 +35,6 @@ final class TTSViewModel: ObservableObject, Loggable { self.availableVoiceIds = synthesizer.config.defaultLanguage .flatMap { voicesByLanguage[$0]?.map { $0.identifier } } ?? [] - self.rateMultiplierRange = synthesizer.rateMultiplierRange - self.pitchMultiplierRange = synthesizer.pitchMultiplierRange } } From b46bbe77a58c4907741f492ff960373d69a83b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sat, 13 Aug 2022 14:05:16 +0200 Subject: [PATCH 41/46] Add the TTS user guide --- CHANGELOG.md | 6 + Documentation/TTS.md | 185 ++++++++++++++++++ .../TTS/PublicationSpeechSynthesizer.swift | 2 - 3 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 Documentation/TTS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c14180a8..d7ff50f36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ All notable changes to this project will be documented in this file. Take a look ## [Unreleased] +### Added + +#### Navigator + +* [A brand new text-to-speech implementation](Documentation/Guides/tts.md). + ### Deprecated #### Shared diff --git a/Documentation/TTS.md b/Documentation/TTS.md new file mode 100644 index 000000000..a22c062bd --- /dev/null +++ b/Documentation/TTS.md @@ -0,0 +1,185 @@ +# Text-to-speech + +:warning: TTS is an experimental feature which is not yet implemented for all formats. + +Text-to-speech can be used to read aloud a publication using a synthetic voice. The Readium toolkit ships with a TTS implementation based on the native [Apple Speech Synthesis](https://developer.apple.com/documentation/avfoundation/speech_synthesis), but it is opened for extension if you want to use a different TTS engine. + +## Glossary + +* **engine** – a TTS engine takes an utterance and transforms it into audio using a synthetic voice +* **tokenizer** - algorithm splitting the publication text content into individual utterances, usually by sentences +* **utterance** - a single piece of text played by a TTS engine, such as a sentence +* **voice** – a synthetic voice is used by a TTS engine to speak a text using rules pertaining to the voice's language and region + +## Reading a publication aloud + +To read a publication, you need to create an instance of `PublicationSpeechSynthesizer`. It orchestrates the rendition of a publication by iterating through its content, splitting it into individual utterances using a `ContentTokenizer`, then using a `TTSEngine` to read them aloud. Not all publications can be read using TTS, therefore the constructor returns an optional object. You can also check whether a publication can be played beforehand using `PublicationSpeechSynthesizer.canSpeak(publication:)`. + +```swift +let synthesizer = PublicationSpeechSynthesizer( + publication: publication, + config: PublicationSpeechSynthesizer.Configuration( + defaultLanguage: Language("fr") + ) +) +``` + +Then, begin the playback from a given starting `Locator`. When missing, the playback will start from the beginning of the publication. + +```swift +synthesizer.start() +``` + +You should now hear the TTS engine speak the utterances from the beginning. `PublicationSpeechSynthesizer` provides the APIs necessary to control the playback from the app: + +* `stop()` - stops the playback ; requires start to be called again +* `pause()` - interrupts the playback temporarily +* `resume()` - resumes the playback where it was paused +* `pauseOrResume()` - toggles the pause +* `previous()` - skips to the previous utterance +* `next()` - skips to the next utterance + +Look at `TTSView.swift` in the Test App for an example of a view calling these APIs. + +## Observing the playback state + +The `PublicationSpeechSynthesizer` should be the single source of truth to represent the playback state in your user interface. You can observe the state with `PublicationSpeechSynthesizerDelegate.publicationSpeechSynthesizer(_:stateDidChange:)` to keep your user interface synchronized with the playback. The possible states are: + +* `.stopped` when idle and waiting for a call to `start()`. +* `.paused(Utterance)` when interrupted while playing the associated utterance. +* `.playing(Utterance, range: Locator?)` when speaking the associated utterance. This state is updated repeatedly while the utterance is spoken, updating the `range` value with the portion of utterance being played (usually the current word). + +When pairing the `PublicationSpeechSynthesizer` with a `Navigator`, you can use the `utterance.locator` and `range` properties to highlight spoken utterances and turn pages automatically. + +## Configuring the TTS + +:warning: The way the synthesizer is configured is expected to change with the introduction of the new Settings API. Expect some breaking changes when updating. + +The `PublicationSpeechSynthesizer` offers some options to configure the TTS engine. Note that the support of each configuration option depends on the TTS engine used. + +Update the configuration by setting it directly. The configuration is not applied right away but for the next utterance. + +```swift +synthesizer.config.defaultLanguage = Language("fr") +``` + +### Default language + +The language used by the synthesizer is important, as it determines which TTS voices are used and the rules to tokenize the publication text content. + +By default, `PublicationSpeechSynthesizer` will use any language explicitly set on a text element (e.g. with `lang="fr"` in HTML) and fall back on the global language declared in the publication manifest. You can override the fallback language with `Configuration.defaultLanguage` which is useful when the publication language is incorrect or missing. + +### Voice + +The `voice` setting can be used to change the synthetic voice used by the engine. To get the available list, use `synthesizer.availableVoices`. + +To restore a user-selected voice, persist the unique voice identifier returned by `voice.identifier`. + +Users do not expect to see all available voices at all time, as they depend on the selected language. You can group the voices by their language and filter them by the selected language using the following snippet. + +```swift +let voicesByLanguage: [Language: [TTSVoice]] = + Dictionary(grouping: synthesizer.availableVoices, by: \.language) +``` + +## Synchronizing the TTS with a Navigator + +While `PublicationSpeechSynthesizer` is completely independent from `Navigator` and can be used to play a publication in the background, most apps prefer to render the publication while it is being read aloud. The `Locator` core model is used as a means to synchronize the synthesizer with the navigator. + +### Starting the TTS from the visible page + +`PublicationSpeechSynthesizer.start()` takes a starting `Locator` for parameter. You can use it to begin the playback from the currently visible page in a `VisualNavigator` using `firstVisibleElementLocator()`. + +```swift +navigator.firstVisibleElementLocator { start in + synthesizer.start(from: start) +} +``` + +### Highlighting the currently spoken utterance + +If you want to highlight or underline the current utterance on the page, you can apply a `Decoration` on the utterance locator with a `DecorableNavigator`. + +```swift +extension TTSViewModel: PublicationSpeechSynthesizerDelegate { + + public func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange synthesizerState: PublicationSpeechSynthesizer.State) { + let playingUtterance: Locator? + + switch synthesizerState { + case .stopped: + playingUtterance = nil + case let .playing(utterance, range: _): + playingUtterance = utterance + case let .paused(utterance): + playingUtterance = utterance + } + + var decorations: [Decoration] = [] + if let locator = playingUtterance.locator { + decorations.append(Decoration( + id: "tts-utterance", + locator: locator, + style: .highlight(tint: .red) + )) + } + navigator.apply(decorations: decorations, in: "tts") + } +} +``` + +### Turning pages automatically + +You can use the same technique as described above to automatically synchronize the `Navigator` with the played utterance, using `navigator.go(to: utterance.locator)`. + +However, this will not turn pages mid-utterance, which can be annoying when speaking a long sentence spanning two pages. To address this, you can use the `range` associated value of the `.playing` state instead. It is updated regularly while speaking each word of an utterance. Note that jumping to the `range` locator for every word can severely impact performances. To alleviate this, you can throttle the observer. + +```swift +extension TTSViewModel: PublicationSpeechSynthesizerDelegate { + + public func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange synthesizerState: PublicationSpeechSynthesizer.State) { + switch synthesizerState { + case .stopped, .paused: + break + case let .playing(_, range: range): + // TODO: You should use throttling here, for example with Combine: + // https://developer.apple.com/documentation/combine/fail/throttle(for:scheduler:latest:) + navigator.go(to: range) + } + } +} +``` + +## Using a custom utterance tokenizer + +By default, the `PublicationSpeechSynthesizer` will split the publication text into sentences to create the utterances. You can customize this for finer or coarser utterances using a different tokenizer. + +For example, this will speak the content word-by-word: + +```swift +let synthesizer = PublicationSpeechSynthesizer( + publication: publication, + tokenizerFactory: { language in + makeTextContentTokenizer( + defaultLanguage: language, + textTokenizerFactory: { language in + makeDefaultTextTokenizer(unit: .word, language: language) + } + ) + } +) +``` + +For completely custom tokenizing or to improve the existing tokenizers, you can implement your own `ContentTokenizer`. + +## Using a custom TTS engine + +`PublicationSpeechSynthesizer` can be used with any TTS engine, provided they implement the `TTSEngine` interface. Take a look at `AVTTSEngine` for an example implementation. + +```swift +let synthesizer = PublicationSpeechSynthesizer( + publication: publication, + engineFactory: { MyCustomEngine() } +) +``` + diff --git a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift index 5b37941e6..2d20c31d7 100644 --- a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift +++ b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift @@ -17,8 +17,6 @@ public protocol PublicationSpeechSynthesizerDelegate: AnyObject { /// `PublicationSpeechSynthesizer` orchestrates the rendition of a `Publication` by iterating through its content, /// splitting it into individual utterances using a `ContentTokenizer`, then using a `TTSEngine` to read them aloud. -/// -/// Don't forget to call `close()` when you are done using the `PublicationSpeechSynthesizer`. public class PublicationSpeechSynthesizer: Loggable { public typealias EngineFactory = () -> TTSEngine public typealias TokenizerFactory = (_ defaultLanguage: Language?) -> ContentTokenizer From be2225d909b470e27a0831306895260acd4fbcbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sat, 13 Aug 2022 14:41:37 +0200 Subject: [PATCH 42/46] Add `Content` helpers --- Documentation/TTS.md | 185 ------------------ .../TTS/PublicationSpeechSynthesizer.swift | 2 +- .../Services/Content/Content.swift | 62 +++++- .../Services/Content/ContentService.swift | 2 +- 4 files changed, 62 insertions(+), 189 deletions(-) delete mode 100644 Documentation/TTS.md diff --git a/Documentation/TTS.md b/Documentation/TTS.md deleted file mode 100644 index a22c062bd..000000000 --- a/Documentation/TTS.md +++ /dev/null @@ -1,185 +0,0 @@ -# Text-to-speech - -:warning: TTS is an experimental feature which is not yet implemented for all formats. - -Text-to-speech can be used to read aloud a publication using a synthetic voice. The Readium toolkit ships with a TTS implementation based on the native [Apple Speech Synthesis](https://developer.apple.com/documentation/avfoundation/speech_synthesis), but it is opened for extension if you want to use a different TTS engine. - -## Glossary - -* **engine** – a TTS engine takes an utterance and transforms it into audio using a synthetic voice -* **tokenizer** - algorithm splitting the publication text content into individual utterances, usually by sentences -* **utterance** - a single piece of text played by a TTS engine, such as a sentence -* **voice** – a synthetic voice is used by a TTS engine to speak a text using rules pertaining to the voice's language and region - -## Reading a publication aloud - -To read a publication, you need to create an instance of `PublicationSpeechSynthesizer`. It orchestrates the rendition of a publication by iterating through its content, splitting it into individual utterances using a `ContentTokenizer`, then using a `TTSEngine` to read them aloud. Not all publications can be read using TTS, therefore the constructor returns an optional object. You can also check whether a publication can be played beforehand using `PublicationSpeechSynthesizer.canSpeak(publication:)`. - -```swift -let synthesizer = PublicationSpeechSynthesizer( - publication: publication, - config: PublicationSpeechSynthesizer.Configuration( - defaultLanguage: Language("fr") - ) -) -``` - -Then, begin the playback from a given starting `Locator`. When missing, the playback will start from the beginning of the publication. - -```swift -synthesizer.start() -``` - -You should now hear the TTS engine speak the utterances from the beginning. `PublicationSpeechSynthesizer` provides the APIs necessary to control the playback from the app: - -* `stop()` - stops the playback ; requires start to be called again -* `pause()` - interrupts the playback temporarily -* `resume()` - resumes the playback where it was paused -* `pauseOrResume()` - toggles the pause -* `previous()` - skips to the previous utterance -* `next()` - skips to the next utterance - -Look at `TTSView.swift` in the Test App for an example of a view calling these APIs. - -## Observing the playback state - -The `PublicationSpeechSynthesizer` should be the single source of truth to represent the playback state in your user interface. You can observe the state with `PublicationSpeechSynthesizerDelegate.publicationSpeechSynthesizer(_:stateDidChange:)` to keep your user interface synchronized with the playback. The possible states are: - -* `.stopped` when idle and waiting for a call to `start()`. -* `.paused(Utterance)` when interrupted while playing the associated utterance. -* `.playing(Utterance, range: Locator?)` when speaking the associated utterance. This state is updated repeatedly while the utterance is spoken, updating the `range` value with the portion of utterance being played (usually the current word). - -When pairing the `PublicationSpeechSynthesizer` with a `Navigator`, you can use the `utterance.locator` and `range` properties to highlight spoken utterances and turn pages automatically. - -## Configuring the TTS - -:warning: The way the synthesizer is configured is expected to change with the introduction of the new Settings API. Expect some breaking changes when updating. - -The `PublicationSpeechSynthesizer` offers some options to configure the TTS engine. Note that the support of each configuration option depends on the TTS engine used. - -Update the configuration by setting it directly. The configuration is not applied right away but for the next utterance. - -```swift -synthesizer.config.defaultLanguage = Language("fr") -``` - -### Default language - -The language used by the synthesizer is important, as it determines which TTS voices are used and the rules to tokenize the publication text content. - -By default, `PublicationSpeechSynthesizer` will use any language explicitly set on a text element (e.g. with `lang="fr"` in HTML) and fall back on the global language declared in the publication manifest. You can override the fallback language with `Configuration.defaultLanguage` which is useful when the publication language is incorrect or missing. - -### Voice - -The `voice` setting can be used to change the synthetic voice used by the engine. To get the available list, use `synthesizer.availableVoices`. - -To restore a user-selected voice, persist the unique voice identifier returned by `voice.identifier`. - -Users do not expect to see all available voices at all time, as they depend on the selected language. You can group the voices by their language and filter them by the selected language using the following snippet. - -```swift -let voicesByLanguage: [Language: [TTSVoice]] = - Dictionary(grouping: synthesizer.availableVoices, by: \.language) -``` - -## Synchronizing the TTS with a Navigator - -While `PublicationSpeechSynthesizer` is completely independent from `Navigator` and can be used to play a publication in the background, most apps prefer to render the publication while it is being read aloud. The `Locator` core model is used as a means to synchronize the synthesizer with the navigator. - -### Starting the TTS from the visible page - -`PublicationSpeechSynthesizer.start()` takes a starting `Locator` for parameter. You can use it to begin the playback from the currently visible page in a `VisualNavigator` using `firstVisibleElementLocator()`. - -```swift -navigator.firstVisibleElementLocator { start in - synthesizer.start(from: start) -} -``` - -### Highlighting the currently spoken utterance - -If you want to highlight or underline the current utterance on the page, you can apply a `Decoration` on the utterance locator with a `DecorableNavigator`. - -```swift -extension TTSViewModel: PublicationSpeechSynthesizerDelegate { - - public func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange synthesizerState: PublicationSpeechSynthesizer.State) { - let playingUtterance: Locator? - - switch synthesizerState { - case .stopped: - playingUtterance = nil - case let .playing(utterance, range: _): - playingUtterance = utterance - case let .paused(utterance): - playingUtterance = utterance - } - - var decorations: [Decoration] = [] - if let locator = playingUtterance.locator { - decorations.append(Decoration( - id: "tts-utterance", - locator: locator, - style: .highlight(tint: .red) - )) - } - navigator.apply(decorations: decorations, in: "tts") - } -} -``` - -### Turning pages automatically - -You can use the same technique as described above to automatically synchronize the `Navigator` with the played utterance, using `navigator.go(to: utterance.locator)`. - -However, this will not turn pages mid-utterance, which can be annoying when speaking a long sentence spanning two pages. To address this, you can use the `range` associated value of the `.playing` state instead. It is updated regularly while speaking each word of an utterance. Note that jumping to the `range` locator for every word can severely impact performances. To alleviate this, you can throttle the observer. - -```swift -extension TTSViewModel: PublicationSpeechSynthesizerDelegate { - - public func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange synthesizerState: PublicationSpeechSynthesizer.State) { - switch synthesizerState { - case .stopped, .paused: - break - case let .playing(_, range: range): - // TODO: You should use throttling here, for example with Combine: - // https://developer.apple.com/documentation/combine/fail/throttle(for:scheduler:latest:) - navigator.go(to: range) - } - } -} -``` - -## Using a custom utterance tokenizer - -By default, the `PublicationSpeechSynthesizer` will split the publication text into sentences to create the utterances. You can customize this for finer or coarser utterances using a different tokenizer. - -For example, this will speak the content word-by-word: - -```swift -let synthesizer = PublicationSpeechSynthesizer( - publication: publication, - tokenizerFactory: { language in - makeTextContentTokenizer( - defaultLanguage: language, - textTokenizerFactory: { language in - makeDefaultTextTokenizer(unit: .word, language: language) - } - ) - } -) -``` - -For completely custom tokenizing or to improve the existing tokenizers, you can implement your own `ContentTokenizer`. - -## Using a custom TTS engine - -`PublicationSpeechSynthesizer` can be used with any TTS engine, provided they implement the `TTSEngine` interface. Take a look at `AVTTSEngine` for an example implementation. - -```swift -let synthesizer = PublicationSpeechSynthesizer( - publication: publication, - engineFactory: { MyCustomEngine() } -) -``` - diff --git a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift index 2d20c31d7..910b207d2 100644 --- a/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift +++ b/Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift @@ -152,7 +152,7 @@ public class PublicationSpeechSynthesizer: Loggable { /// (Re)starts the synthesizer from the given locator or the beginning of the publication. public func start(from startLocator: Locator? = nil) { currentTask?.cancel() - publicationIterator = publication.content(from: startLocator)?.makeIterator() + publicationIterator = publication.content(from: startLocator)?.iterator() playNextUtterance(.forward) } diff --git a/Sources/Shared/Publication/Services/Content/Content.swift b/Sources/Shared/Publication/Services/Content/Content.swift index 2d5bb0f65..dc513b169 100644 --- a/Sources/Shared/Publication/Services/Content/Content.swift +++ b/Sources/Shared/Publication/Services/Content/Content.swift @@ -8,7 +8,35 @@ import Foundation /// Provides an iterable list of `ContentElement`s. public protocol Content { - func makeIterator() -> ContentIterator + /// Creates a new fallible bidirectional iterator for this content. + func iterator() -> ContentIterator +} + +public extension Content { + /// Returns a `Sequence` of all elements. + func sequence() -> ContentSequence { + ContentSequence(content: self) + } + + /// Returns all the elements as a list. + func elements() -> [ContentElement] { + Array(sequence()) + } + + /// Extracts the full raw text, or returns null if no text content can be found. + /// + /// - Parameter separator: Separator to use between individual elements. Defaults to newline. + func text(separator: String = "\n") -> String? { + let text = elements() + .compactMap { ($0 as? TextualContentElement)?.text.takeIf { !$0.isEmpty } } + .joined(separator: separator) + + guard !text.isEmpty else { + return nil + } + + return text + } } /// Represents a single semantic content element part of a publication. @@ -213,4 +241,34 @@ public protocol ContentIterator: AnyObject { /// Advances to the previous item and returns it, or null if we reached the beginning. func previous() throws -> ContentElement? -} \ No newline at end of file +} + +/// Helper class to treat a `Content` as a `Sequence`. +public class ContentSequence: Sequence { + private let content: Content + + init(content: Content) { + self.content = content + } + + public func makeIterator() -> ContentSequence.Iterator { + Iterator(iterator: content.iterator()) + } + + public class Iterator: IteratorProtocol, Loggable { + private let iterator: ContentIterator + + public init(iterator: ContentIterator) { + self.iterator = iterator + } + + public func next() -> ContentElement? { + do { + return try iterator.next() + } catch { + log(.warning, error) + return next() + } + } + } +} diff --git a/Sources/Shared/Publication/Services/Content/ContentService.swift b/Sources/Shared/Publication/Services/Content/ContentService.swift index 4aaac3c1e..7a0c72205 100644 --- a/Sources/Shared/Publication/Services/Content/ContentService.swift +++ b/Sources/Shared/Publication/Services/Content/ContentService.swift @@ -51,7 +51,7 @@ public class DefaultContentService: ContentService { self.resourceContentIteratorFactories = resourceContentIteratorFactories } - func makeIterator() -> ContentIterator { + func iterator() -> ContentIterator { PublicationContentIterator( publication: publication, start: start, From 31dc9f89c91546db08b1f7b7bea17fc36015d0c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sat, 13 Aug 2022 15:18:33 +0200 Subject: [PATCH 43/46] Add the content user guide --- CHANGELOG.md | 6 +- Documentation/Guides/Content.md | 202 ++++++++++++++++++++++++++++++++ Documentation/Guides/TTS.md | 185 +++++++++++++++++++++++++++++ 3 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 Documentation/Guides/Content.md create mode 100644 Documentation/Guides/TTS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d7ff50f36..a5e0a9242 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,13 @@ All notable changes to this project will be documented in this file. Take a look ### Added +#### Shared + +* [Extract the raw content (text, images, etc.) of a publication](Documentation/Guides/Content.md). + #### Navigator -* [A brand new text-to-speech implementation](Documentation/Guides/tts.md). +* [A brand new text-to-speech implementation](Documentation/Guides/TTS.md). ### Deprecated diff --git a/Documentation/Guides/Content.md b/Documentation/Guides/Content.md new file mode 100644 index 000000000..5dddb65af --- /dev/null +++ b/Documentation/Guides/Content.md @@ -0,0 +1,202 @@ +# Extracting the content of a publication + +:warning: The described feature is still experimental and the implementation incomplete. + +Many high-level features require access to the raw content (text, media, etc.) of a publication, such as: + +* Text-to-speech +* Accessibility reader +* Basic search +* Full-text search indexing +* Image or audio indexes + +The `ContentService` provides a way to iterate through a publication's content, extracted as semantic elements. + +First, request the publication's `Content`, starting from a given `Locator`. If the locator is missing, the `Content` will be extracted from the beginning of the publication. + +```swift +guard let content = publication.content(from: startLocator) else { + // Abort as the content cannot be extracted + return +} +``` + +## Extracting the raw text content + +Getting the whole raw text of a publication is such a common use case that a helper is available on `Content`: + +```swift +let wholeText = content.text() +``` + +This is an expensive operation, proceed with caution and cache the result if you need to reuse it. + +## Iterating through the content + +The individual `Content` elements can be iterated through with a regular `for` loop by converting it to a sequence: + +```swift +for (element in content.sequence()) { + // Process element +} +``` + +Alternatively, you can get the whole list of elements with `content.elements()`, or use the lower level APIs to iterate the content manually: + +```swift +let iterator = content.iterator() +while let element = try iterator.next() { + print(element) +} +``` + +Some `Content` implementations support bidirectional iterations. To iterate backwards, use: + +```swift +let iterator = content.iterator() +while let element = try iterator.previous() { + print(element) +} +``` + +## Processing the elements + +The `Content` iterator yields `ContentElement` objects representing a single semantic portion of the publication, such as a heading, a paragraph or an embedded image. + +Every element has a `locator` property targeting it in the publication. You can use the locator, for example, to navigate to the element or to draw a `Decoration` on top of it. + +```swift +navigator.go(to: element.locator) +``` + +### Types of elements + +Depending on the actual implementation of `ContentElement`, more properties are available to access the actual data. The toolkit ships with a number of default implementations for common types of elements. + +#### Embedded media + +The `EmbeddedContentElement` protocol is implemented by any element referencing an external resource. It contains an `embeddedLink` property you can use to get the actual content of the resource. + +```swift +if let element = element as? EmbeddedContentElement { + let bytes = try publication + .get(element.embeddedLink) + .read().get() +} +``` + +Here are the default available implementations: + +* `AudioContentElement` - audio clips +* `VideoContentElement` - video clips +* `ImageContentElement` - bitmap images, with the additional property: + * `caption: String?` - figure caption, when available + +#### Text + +##### Textual elements + +The `TextualContentElement` protocol is implemented by any element which can be represented as human-readable text. This is useful when you want to extract the text content of a publication without caring for each individual type of elements. + +```swift +let wholeText = publication.content() + .elements() + .compactMap { ($0 as? TextualContentElement)?.text.takeIf { !$0.isEmpty } } + .joined(separator: "\n") +``` + +##### Text elements + +Actual text elements are instances of `TextContentElement`, which represent a single block of text such as a heading, a paragraph or a list item. It is comprised of a `role` and a list of `segments`. + +The `role` is the nature of the text element in the document. For example a heading, body, footnote or a quote. It can be used to reconstruct part of the structure of the original document. + +A text element is composed of individual segments with their own `locator` and `attributes`. They are useful to associate attributes with a portion of a text element. For example, given the HTML paragraph: + +```html +

It is pronounced croissant.

+``` + +The following `TextContentElement` will be produced: + +```swift +TextContentElement( + role: .body, + segments: [ + TextContentElement.Segment(text: "It is pronounced "), + TextContentElement.Segment(text: "croissant", attributes: [ContentAttribute(key: .language, value: "fr")]), + TextContentElement.Segment(text: ".") + ] +) +``` + +If you are not interested in the segment attributes, you can also use `element.text` to get the concatenated raw text. + +### Element attributes + +All types of `ContentElement` can have associated attributes. Custom `ContentService` implementations can use this as an extensibility point. + +## Use cases + +### An index of all images embedded in the publication + +This example extracts all the embedded images in the publication and displays them in a SwiftUI list. Clicking on an image jumps to its location in the publication. + +```swift +struct ImageIndex: View { + struct Item: Hashable { + let locator: Locator + let text: String? + let image: UIImage + } + + let publication: Publication + let navigator: Navigator + @State private var items: [Item] = [] + + init(publication: Publication, navigator: Navigator) { + self.publication = publication + self.navigator = navigator + } + + var body: some View { + ScrollView { + LazyVStack { + ForEach(items, id: \.self) { item in + VStack() { + Image(uiImage: item.image) + Text(item.text ?? "No caption") + } + .onTapGesture { + navigator.go(to: item.locator) + } + } + } + } + .onAppear { + items = publication.content()? + .elements() + .compactMap { element in + guard + let element = element as? ImageContentElement, + let image = try? publication.get(element.embeddedLink) + .read().map(UIImage.init).get() + else { + return nil + } + + return Item( + locator: element.locator, + text: element.caption ?? element.accessibilityLabel, + image: image + ) + } + ?? [] + } + } +} +``` + +## References + +* [Content Iterator proposal](https://github.com/readium/architecture/pull/177) diff --git a/Documentation/Guides/TTS.md b/Documentation/Guides/TTS.md new file mode 100644 index 000000000..a22c062bd --- /dev/null +++ b/Documentation/Guides/TTS.md @@ -0,0 +1,185 @@ +# Text-to-speech + +:warning: TTS is an experimental feature which is not yet implemented for all formats. + +Text-to-speech can be used to read aloud a publication using a synthetic voice. The Readium toolkit ships with a TTS implementation based on the native [Apple Speech Synthesis](https://developer.apple.com/documentation/avfoundation/speech_synthesis), but it is opened for extension if you want to use a different TTS engine. + +## Glossary + +* **engine** – a TTS engine takes an utterance and transforms it into audio using a synthetic voice +* **tokenizer** - algorithm splitting the publication text content into individual utterances, usually by sentences +* **utterance** - a single piece of text played by a TTS engine, such as a sentence +* **voice** – a synthetic voice is used by a TTS engine to speak a text using rules pertaining to the voice's language and region + +## Reading a publication aloud + +To read a publication, you need to create an instance of `PublicationSpeechSynthesizer`. It orchestrates the rendition of a publication by iterating through its content, splitting it into individual utterances using a `ContentTokenizer`, then using a `TTSEngine` to read them aloud. Not all publications can be read using TTS, therefore the constructor returns an optional object. You can also check whether a publication can be played beforehand using `PublicationSpeechSynthesizer.canSpeak(publication:)`. + +```swift +let synthesizer = PublicationSpeechSynthesizer( + publication: publication, + config: PublicationSpeechSynthesizer.Configuration( + defaultLanguage: Language("fr") + ) +) +``` + +Then, begin the playback from a given starting `Locator`. When missing, the playback will start from the beginning of the publication. + +```swift +synthesizer.start() +``` + +You should now hear the TTS engine speak the utterances from the beginning. `PublicationSpeechSynthesizer` provides the APIs necessary to control the playback from the app: + +* `stop()` - stops the playback ; requires start to be called again +* `pause()` - interrupts the playback temporarily +* `resume()` - resumes the playback where it was paused +* `pauseOrResume()` - toggles the pause +* `previous()` - skips to the previous utterance +* `next()` - skips to the next utterance + +Look at `TTSView.swift` in the Test App for an example of a view calling these APIs. + +## Observing the playback state + +The `PublicationSpeechSynthesizer` should be the single source of truth to represent the playback state in your user interface. You can observe the state with `PublicationSpeechSynthesizerDelegate.publicationSpeechSynthesizer(_:stateDidChange:)` to keep your user interface synchronized with the playback. The possible states are: + +* `.stopped` when idle and waiting for a call to `start()`. +* `.paused(Utterance)` when interrupted while playing the associated utterance. +* `.playing(Utterance, range: Locator?)` when speaking the associated utterance. This state is updated repeatedly while the utterance is spoken, updating the `range` value with the portion of utterance being played (usually the current word). + +When pairing the `PublicationSpeechSynthesizer` with a `Navigator`, you can use the `utterance.locator` and `range` properties to highlight spoken utterances and turn pages automatically. + +## Configuring the TTS + +:warning: The way the synthesizer is configured is expected to change with the introduction of the new Settings API. Expect some breaking changes when updating. + +The `PublicationSpeechSynthesizer` offers some options to configure the TTS engine. Note that the support of each configuration option depends on the TTS engine used. + +Update the configuration by setting it directly. The configuration is not applied right away but for the next utterance. + +```swift +synthesizer.config.defaultLanguage = Language("fr") +``` + +### Default language + +The language used by the synthesizer is important, as it determines which TTS voices are used and the rules to tokenize the publication text content. + +By default, `PublicationSpeechSynthesizer` will use any language explicitly set on a text element (e.g. with `lang="fr"` in HTML) and fall back on the global language declared in the publication manifest. You can override the fallback language with `Configuration.defaultLanguage` which is useful when the publication language is incorrect or missing. + +### Voice + +The `voice` setting can be used to change the synthetic voice used by the engine. To get the available list, use `synthesizer.availableVoices`. + +To restore a user-selected voice, persist the unique voice identifier returned by `voice.identifier`. + +Users do not expect to see all available voices at all time, as they depend on the selected language. You can group the voices by their language and filter them by the selected language using the following snippet. + +```swift +let voicesByLanguage: [Language: [TTSVoice]] = + Dictionary(grouping: synthesizer.availableVoices, by: \.language) +``` + +## Synchronizing the TTS with a Navigator + +While `PublicationSpeechSynthesizer` is completely independent from `Navigator` and can be used to play a publication in the background, most apps prefer to render the publication while it is being read aloud. The `Locator` core model is used as a means to synchronize the synthesizer with the navigator. + +### Starting the TTS from the visible page + +`PublicationSpeechSynthesizer.start()` takes a starting `Locator` for parameter. You can use it to begin the playback from the currently visible page in a `VisualNavigator` using `firstVisibleElementLocator()`. + +```swift +navigator.firstVisibleElementLocator { start in + synthesizer.start(from: start) +} +``` + +### Highlighting the currently spoken utterance + +If you want to highlight or underline the current utterance on the page, you can apply a `Decoration` on the utterance locator with a `DecorableNavigator`. + +```swift +extension TTSViewModel: PublicationSpeechSynthesizerDelegate { + + public func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange synthesizerState: PublicationSpeechSynthesizer.State) { + let playingUtterance: Locator? + + switch synthesizerState { + case .stopped: + playingUtterance = nil + case let .playing(utterance, range: _): + playingUtterance = utterance + case let .paused(utterance): + playingUtterance = utterance + } + + var decorations: [Decoration] = [] + if let locator = playingUtterance.locator { + decorations.append(Decoration( + id: "tts-utterance", + locator: locator, + style: .highlight(tint: .red) + )) + } + navigator.apply(decorations: decorations, in: "tts") + } +} +``` + +### Turning pages automatically + +You can use the same technique as described above to automatically synchronize the `Navigator` with the played utterance, using `navigator.go(to: utterance.locator)`. + +However, this will not turn pages mid-utterance, which can be annoying when speaking a long sentence spanning two pages. To address this, you can use the `range` associated value of the `.playing` state instead. It is updated regularly while speaking each word of an utterance. Note that jumping to the `range` locator for every word can severely impact performances. To alleviate this, you can throttle the observer. + +```swift +extension TTSViewModel: PublicationSpeechSynthesizerDelegate { + + public func publicationSpeechSynthesizer(_ synthesizer: PublicationSpeechSynthesizer, stateDidChange synthesizerState: PublicationSpeechSynthesizer.State) { + switch synthesizerState { + case .stopped, .paused: + break + case let .playing(_, range: range): + // TODO: You should use throttling here, for example with Combine: + // https://developer.apple.com/documentation/combine/fail/throttle(for:scheduler:latest:) + navigator.go(to: range) + } + } +} +``` + +## Using a custom utterance tokenizer + +By default, the `PublicationSpeechSynthesizer` will split the publication text into sentences to create the utterances. You can customize this for finer or coarser utterances using a different tokenizer. + +For example, this will speak the content word-by-word: + +```swift +let synthesizer = PublicationSpeechSynthesizer( + publication: publication, + tokenizerFactory: { language in + makeTextContentTokenizer( + defaultLanguage: language, + textTokenizerFactory: { language in + makeDefaultTextTokenizer(unit: .word, language: language) + } + ) + } +) +``` + +For completely custom tokenizing or to improve the existing tokenizers, you can implement your own `ContentTokenizer`. + +## Using a custom TTS engine + +`PublicationSpeechSynthesizer` can be used with any TTS engine, provided they implement the `TTSEngine` interface. Take a look at `AVTTSEngine` for an example implementation. + +```swift +let synthesizer = PublicationSpeechSynthesizer( + publication: publication, + engineFactory: { MyCustomEngine() } +) +``` + From bc4cbea17bb0529c3198a4768b0fc0a3d5c96f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sat, 13 Aug 2022 15:23:01 +0200 Subject: [PATCH 44/46] Update Carthage project --- Support/Carthage/.xcodegen | 20 +++++ .../Readium.xcodeproj/project.pbxproj | 88 +++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/Support/Carthage/.xcodegen b/Support/Carthage/.xcodegen index 6b2e0dd4b..40bd35200 100644 --- a/Support/Carthage/.xcodegen +++ b/Support/Carthage/.xcodegen @@ -10553,6 +10553,7 @@ ../../Sources/Navigator/EPUB/Scripts/README.md ../../Sources/Navigator/EPUB/Scripts/src ../../Sources/Navigator/EPUB/Scripts/src/decorator.js +../../Sources/Navigator/EPUB/Scripts/src/dom.js ../../Sources/Navigator/EPUB/Scripts/src/fixed-page.js ../../Sources/Navigator/EPUB/Scripts/src/gestures.js ../../Sources/Navigator/EPUB/Scripts/src/index-fixed-wrapper-one.js @@ -10606,9 +10607,11 @@ ../../Sources/Navigator/SelectableNavigator.swift ../../Sources/Navigator/Toolkit ../../Sources/Navigator/Toolkit/CompletionList.swift +../../Sources/Navigator/Toolkit/CursorList.swift ../../Sources/Navigator/Toolkit/Extensions ../../Sources/Navigator/Toolkit/Extensions/Bundle.swift ../../Sources/Navigator/Toolkit/Extensions/CGRect.swift +../../Sources/Navigator/Toolkit/Extensions/Range.swift ../../Sources/Navigator/Toolkit/Extensions/UIColor.swift ../../Sources/Navigator/Toolkit/Extensions/UIView.swift ../../Sources/Navigator/Toolkit/Extensions/WKWebView.swift @@ -10616,6 +10619,10 @@ ../../Sources/Navigator/Toolkit/R2NavigatorLocalizedString.swift ../../Sources/Navigator/Toolkit/TargetAction.swift ../../Sources/Navigator/Toolkit/WebView.swift +../../Sources/Navigator/TTS +../../Sources/Navigator/TTS/AVTTSEngine.swift +../../Sources/Navigator/TTS/PublicationSpeechSynthesizer.swift +../../Sources/Navigator/TTS/TTSEngine.swift ../../Sources/Navigator/VisualNavigator.swift ../../Sources/OPDS ../../Sources/OPDS/Deprecated.swift @@ -10702,10 +10709,17 @@ ../../Sources/Shared/Publication/PublicationCollection.swift ../../Sources/Shared/Publication/ReadingProgression.swift ../../Sources/Shared/Publication/Services +../../Sources/Shared/Publication/Services/Content ../../Sources/Shared/Publication/Services/Content Protection ../../Sources/Shared/Publication/Services/Content Protection/ContentProtectionService.swift ../../Sources/Shared/Publication/Services/Content Protection/ContentProtectionService+WS.swift ../../Sources/Shared/Publication/Services/Content Protection/UserRights.swift +../../Sources/Shared/Publication/Services/Content/Content.swift +../../Sources/Shared/Publication/Services/Content/ContentService.swift +../../Sources/Shared/Publication/Services/Content/ContentTokenizer.swift +../../Sources/Shared/Publication/Services/Content/Iterators +../../Sources/Shared/Publication/Services/Content/Iterators/HTMLResourceContentIterator.swift +../../Sources/Shared/Publication/Services/Content/Iterators/PublicationContentIterator.swift ../../Sources/Shared/Publication/Services/Cover ../../Sources/Shared/Publication/Services/Cover/CoverService.swift ../../Sources/Shared/Publication/Services/Cover/GeneratedCoverService.swift @@ -10730,6 +10744,7 @@ ../../Sources/Shared/Resources/en.lproj/Localizable.strings ../../Sources/Shared/RootFile.swift ../../Sources/Shared/Toolkit +../../Sources/Shared/Toolkit/.DS_Store ../../Sources/Shared/Toolkit/Archive ../../Sources/Shared/Toolkit/Archive/Archive.swift ../../Sources/Shared/Toolkit/Archive/ExplodedArchive.swift @@ -10750,6 +10765,7 @@ ../../Sources/Shared/Toolkit/Extensions/Collection.swift ../../Sources/Shared/Toolkit/Extensions/NSRegularExpression.swift ../../Sources/Shared/Toolkit/Extensions/Optional.swift +../../Sources/Shared/Toolkit/Extensions/Range.swift ../../Sources/Shared/Toolkit/Extensions/Result.swift ../../Sources/Shared/Toolkit/Extensions/String.swift ../../Sources/Shared/Toolkit/Extensions/StringEncoding.swift @@ -10764,6 +10780,7 @@ ../../Sources/Shared/Toolkit/HTTP/HTTPProblemDetails.swift ../../Sources/Shared/Toolkit/HTTP/HTTPRequest.swift ../../Sources/Shared/Toolkit/JSON.swift +../../Sources/Shared/Toolkit/Language.swift ../../Sources/Shared/Toolkit/Logging ../../Sources/Shared/Toolkit/Logging/WarningLogger.swift ../../Sources/Shared/Toolkit/Media @@ -10783,6 +10800,9 @@ ../../Sources/Shared/Toolkit/PDF/PDFOutlineNode.swift ../../Sources/Shared/Toolkit/R2LocalizedString.swift ../../Sources/Shared/Toolkit/ResourcesServer.swift +../../Sources/Shared/Toolkit/Tokenizer +../../Sources/Shared/Toolkit/Tokenizer/TextTokenizer.swift +../../Sources/Shared/Toolkit/Tokenizer/Tokenizer.swift ../../Sources/Shared/Toolkit/URITemplate.swift ../../Sources/Shared/Toolkit/UTI.swift ../../Sources/Shared/Toolkit/Weak.swift diff --git a/Support/Carthage/Readium.xcodeproj/project.pbxproj b/Support/Carthage/Readium.xcodeproj/project.pbxproj index 2390dc5fd..ff09e6644 100644 --- a/Support/Carthage/Readium.xcodeproj/project.pbxproj +++ b/Support/Carthage/Readium.xcodeproj/project.pbxproj @@ -29,6 +29,7 @@ 108D833B59AF7643DB45D867 /* Zip.h in Headers */ = {isa = PBXBuildFile; fileRef = CE641F78FD99A426A80B3495 /* Zip.h */; settings = {ATTRIBUTES = (Public, ); }; }; 1221E200A377D294050B8F00 /* LicenseValidation.swift in Sources */ = {isa = PBXBuildFile; fileRef = BDEFB3D1218817F835A3C5F4 /* LicenseValidation.swift */; }; 134AF2657ABA617255DE2D0A /* Publication.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2DF03272C07D6951ADC1311E /* Publication.swift */; }; + 1399283B7E9E39AADA4EE7DD /* AVTTSEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = B15C9123EA383ED81DE0393A /* AVTTSEngine.swift */; }; 140C2EA93F9215A8F01AB0A3 /* NowPlayingInfo.swift in Sources */ = {isa = PBXBuildFile; fileRef = D6BCDFDD5327AB802F0F6460 /* NowPlayingInfo.swift */; }; 145613F96ED2BC440F2146B8 /* SQLite.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = F07214E263C6589987A561F9 /* SQLite.xcframework */; }; 14B95678D1380759F144B2DF /* EPUBMetadataParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 76E46B10FD5B26A2F41718E0 /* EPUBMetadataParser.swift */; }; @@ -37,6 +38,7 @@ 17CA2D61768F693B8173DBC4 /* PDFParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = D13272E03B63E96D4246F79D /* PDFParser.swift */; }; 18217BC157557A5DDA4BA119 /* User.swift in Sources */ = {isa = PBXBuildFile; fileRef = ADDB8B9906FC78C038203BDD /* User.swift */; }; 185301970A639F99F1C35056 /* Parser+Deprecated.swift in Sources */ = {isa = PBXBuildFile; fileRef = A42F188134C13EB2ECFFB621 /* Parser+Deprecated.swift */; }; + 198089C002038C10FDFBA2BF /* HTMLResourceContentIterator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34AB954525AC159166C96A36 /* HTMLResourceContentIterator.swift */; }; 1BF9469B4574D30E5C9BB75E /* Event.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCF859D4933121BDC376CC8A /* Event.swift */; }; 1CEBFEA40D42C941A49F1A4D /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = 5507BD4012032A7567175B69 /* Localizable.strings */; }; 1E4805B8E562211F264FB16B /* Logger.swift in Sources */ = {isa = PBXBuildFile; fileRef = BD03AFC9C69E785886FB9620 /* Logger.swift */; }; @@ -80,6 +82,7 @@ 3D9CB0E9FD88A14EEF9D7F2A /* ExplodedArchive.swift in Sources */ = {isa = PBXBuildFile; fileRef = C59803AADFCF32C93C9D9D29 /* ExplodedArchive.swift */; }; 3E7614CCBAD233B2D90BF5DC /* PDFPositionsService.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C2A38D366CE8560BCBAC8B /* PDFPositionsService.swift */; }; 3ED6D98B993DB299CFB0513A /* Seekable.swift in Sources */ = {isa = PBXBuildFile; fileRef = A37141BCDFDB6BDBB58CDDD8 /* Seekable.swift */; }; + 411B624A5AE5189875950DDA /* Language.swift in Sources */ = {isa = PBXBuildFile; fileRef = 68FF131876FA3A63025F2662 /* Language.swift */; }; 41D9812679A98F44DA9E7BFD /* UIColor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5E788FD34BE635B4B80C18A6 /* UIColor.swift */; }; 4203767BBAACC7B330623F62 /* Cancellable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BBD54FD376456C1925316BC /* Cancellable.swift */; }; 44152DBECE34F063AD0E93BC /* Link.swift in Sources */ = {isa = PBXBuildFile; fileRef = EE3E6442F0C7FE2098D71F27 /* Link.swift */; }; @@ -94,6 +97,7 @@ 4D4D25BA4772674DD6041C01 /* Deprecated.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0E1D7FA19C628EA8F967F580 /* Deprecated.swift */; }; 4E2AF522FFBD929F52153DAE /* R2Shared.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 41A0528117E270B68AC75C56 /* R2Shared.framework */; }; 4F8168F527F489AB8619A7F1 /* R2Shared.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 41A0528117E270B68AC75C56 /* R2Shared.framework */; }; + 501E7E05DEA11F7A61D60EAF /* Range.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3231F989F7D7E560DD5364B9 /* Range.swift */; }; 502D4ABD63FE9D99AD066F31 /* DOMRange.swift in Sources */ = {isa = PBXBuildFile; fileRef = C084C255A327387F36B97A62 /* DOMRange.swift */; }; 50838C73E245D9F08BFB3159 /* OPDSAcquisition.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3DFAC865449A1A225BF534DA /* OPDSAcquisition.swift */; }; 50ED47D5333272EC72732E42 /* HTMLDecorationTemplate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8AB3B86AB42261727B2811CF /* HTMLDecorationTemplate.swift */; }; @@ -125,6 +129,7 @@ 61FFC793CCF795278998A19E /* SearchService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9B5B029CA09EE1F86A19612A /* SearchService.swift */; }; 635220F58D2B5A0BF8CE4B77 /* Assets in Resources */ = {isa = PBXBuildFile; fileRef = DBCE9786DD346E6BDB2E50FF /* Assets */; }; 650ECC5AC05D337B6A618EBD /* WKWebView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 103E0171A3CDEFA1B1F1F180 /* WKWebView.swift */; }; + 65C6F8A05A0B3ACD2EE44944 /* PublicationSpeechSynthesizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 99DE4955327D8C2DE6F866D3 /* PublicationSpeechSynthesizer.swift */; }; 6719F981514309A65D206A85 /* LCPAcquisition.swift in Sources */ = {isa = PBXBuildFile; fileRef = F622773881411FB8BE686B9F /* LCPAcquisition.swift */; }; 69150D0B00F5665C3DA0000B /* LazyResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C3A9CF25E925418A1712C0B /* LazyResource.swift */; }; 69AA254E4A39D9B49FDFD648 /* UserKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = EC96A56AB406203898059B6C /* UserKey.swift */; }; @@ -153,6 +158,7 @@ 7F0C0E92322B0386DB0911BC /* WebView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 93BF3947EBA8736BF20F36FB /* WebView.swift */; }; 7F297EC335D8934E50361D39 /* ReadiumLicenseContainer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01D191FF1BE0BA97581EB070 /* ReadiumLicenseContainer.swift */; }; 80B2146BDF073A2FF1C28426 /* CGRect.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48856E9AB402E2907B5230F3 /* CGRect.swift */; }; + 812ED3E1480A1D7AA6149F69 /* ContentService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 18E809378D79D09192A0AAE1 /* ContentService.swift */; }; 81ADB258F083647221CED24F /* DataCompression.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1EBC685D4A0E07997088DD2D /* DataCompression.swift */; }; 825642E013351C922B6510AD /* UTI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 48B28C65845F0575C40877F6 /* UTI.swift */; }; 82BAA3EB081DD29A928958AC /* ContentLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = E4A496C959F870BAFDB447DA /* ContentLayout.swift */; }; @@ -176,11 +182,13 @@ 90CFD62B993F6759716C0AF0 /* LicensesService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56286133DD0AE093F2C5E9FD /* LicensesService.swift */; }; 92570B878B678E9E9138C94F /* Links.swift in Sources */ = {isa = PBXBuildFile; fileRef = 64ED7629E73022C1495081D1 /* Links.swift */; }; 95B9369AE4743FB7BAB93DCC /* ResourceContentExtractor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5D8FE0EA948A4FD3AF0DA7D8 /* ResourceContentExtractor.swift */; }; + 95DBB33898FFA425A5913F0C /* ContentTokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 060213D1B559927504AFF9AF /* ContentTokenizer.swift */; }; 97A0F3DC6BEC43D63B80B868 /* RoutingFetcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2DE48021CF3FED1C3340E458 /* RoutingFetcher.swift */; }; 98018A77E2A1FA2B90C987E1 /* AudioParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = D9FFEB1FF4B5CD74EB35CD63 /* AudioParser.swift */; }; 98428BC24846D534B940CE86 /* CryptoSwift.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = E37F94C388A86CB8A34812A5 /* CryptoSwift.xcframework */; }; 98702AFB56F9C50F7246CDDA /* LCPError.swift in Sources */ = {isa = PBXBuildFile; fileRef = A5A115134AA0B8F5254C8139 /* LCPError.swift */; }; 98ABD996FB77EDF7DA69B18F /* DiffableDecoration+HTML.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01265194649A8E2A821CC2A4 /* DiffableDecoration+HTML.swift */; }; + 99856B9FCC56A9F1946C6A60 /* TextTokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 761D7DFCF307078B7283A14E /* TextTokenizer.swift */; }; 99F3C8988EA41B0376D85F72 /* Bundle.swift in Sources */ = {isa = PBXBuildFile; fileRef = FCEE6DBDF8E3D1ABE990DB33 /* Bundle.swift */; }; 9A22C456F6A73F29AD9B0CE8 /* Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 11252900E9B0827C0FD2FA4B /* Database.swift */; }; 9A993922691ACA961F3B16A7 /* DataExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = B2C9762191DAD823E7C925A5 /* DataExtension.swift */; }; @@ -189,6 +197,7 @@ 9BD8989B1CADB1712B31E0A4 /* HREF.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34CA9A244D941CB63515EDDE /* HREF.swift */; }; 9BF0647F4760B562545BC926 /* DataResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = C5B28AF73252B570AEAF80B5 /* DataResource.swift */; }; 9C1DD6AEFB6E1D5989EC25D2 /* Loggable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 067E58BE65BCB4F8D1E8B911 /* Loggable.swift */; }; + 9C682824485E27814F92285F /* CursorList.swift in Sources */ = {isa = PBXBuildFile; fileRef = C361F965E7A7962CA3E4C0BA /* CursorList.swift */; }; 9C6B7AFB6FB0635EF5B7B71C /* JSON.swift in Sources */ = {isa = PBXBuildFile; fileRef = EDA827FC94F5CB3F9032028F /* JSON.swift */; }; 9D0DB30B8FDC56DBFA70E68F /* DocumentTypes.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A00FF0C84822A134A353BD4 /* DocumentTypes.swift */; }; 9DF8A7CF028D764E9D6A2BAC /* DiffableDecoration.swift in Sources */ = {isa = PBXBuildFile; fileRef = 41B61198128D628CFB3FD22A /* DiffableDecoration.swift */; }; @@ -232,8 +241,10 @@ C283E515CA6A8EEA1C89AD98 /* License.swift in Sources */ = {isa = PBXBuildFile; fileRef = C2C93C33347DC0A41FE15AC6 /* License.swift */; }; C2A1FAC4ADA33EABA1E45EF8 /* ParseData.swift in Sources */ = {isa = PBXBuildFile; fileRef = B1085F2D690A73984E675D54 /* ParseData.swift */; }; C2D32286200D850101D8C4FD /* SwiftSoup.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = BE09289EB0FEA5FEC8506B1F /* SwiftSoup.xcframework */; }; + C35001848411CBCAC8F03763 /* PublicationContentIterator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4BF38F71FDEC1920325B62D3 /* PublicationContentIterator.swift */; }; C3BC5A4C44DD8CE26155C0D5 /* PDFFileParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8103346E73760F07800EB75E /* PDFFileParser.swift */; }; C3BEB5CC9C6DD065B2CAE1BE /* Licenses.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED568512FD1304D6B9CC79B0 /* Licenses.swift */; }; + C4AAABD4474B6A5A25B34720 /* Range.swift in Sources */ = {isa = PBXBuildFile; fileRef = AB3E08C8187DCC3099CF9D22 /* Range.swift */; }; C4F0A98562FDDB478F7DD0A9 /* LCPLicense.swift in Sources */ = {isa = PBXBuildFile; fileRef = 093629E752DE17264B97C598 /* LCPLicense.swift */; }; C563FF7E2BDFBD5454067ECD /* EPUBLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = 339637CCF01E665F4CB78B01 /* EPUBLayout.swift */; }; C5D9F9950D332C7CAA0C387A /* ReadiumWebPubParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = E6E97CCA91F910315C260373 /* ReadiumWebPubParser.swift */; }; @@ -244,6 +255,7 @@ C9CD140B788A26AC2604316C /* EPUBDeobfuscator.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3D785FEFDA202A61E620890 /* EPUBDeobfuscator.swift */; }; CA152829D0654EB38D6BF836 /* R2LocalizedString.swift in Sources */ = {isa = PBXBuildFile; fileRef = 972DC46120E457918E69EBD0 /* R2LocalizedString.swift */; }; CC1EBC553CE8A5C873E7A9BB /* PublicationServicesBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7768FC212BAC1669A5ED08C5 /* PublicationServicesBuilder.swift */; }; + CD7DF8DC7B346AA86DECE596 /* Tokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8DA31089FCAD8DFB9AC46E4E /* Tokenizer.swift */; }; CE31AFB76CC1A587AC62BBDB /* EPUBContainerParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78033AFDF98C92351785D17F /* EPUBContainerParser.swift */; }; D13F342C611C6495554EE3DF /* NCXParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4363E8A92B1EA9AF2561DCE9 /* NCXParser.swift */; }; D248F68B569EDADA445E341D /* TargetAction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4E564AE6D5137499C81FEBE2 /* TargetAction.swift */; }; @@ -268,6 +280,7 @@ E3848BCC92B4B7E03A4FEE76 /* EPUBReflowableSpreadView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C45688D0A9C1F81F463FF92 /* EPUBReflowableSpreadView.swift */; }; E55B69F79BB4E2EAC4BE34D0 /* Publication+OPDS.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4BB5D42EEF0083D833E2A572 /* Publication+OPDS.swift */; }; E69BA16E04FBEAA061759895 /* UserSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = F4937644CB65AE6801CE3295 /* UserSettings.swift */; }; + E6B6841AFFF9EFECAEE77ECC /* Content.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1039900AC78465AD989D7464 /* Content.swift */; }; E6BF3A99E6C6AAC4FEE1099F /* ControlFlow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 55BC4119B8937D17ED80B1AB /* ControlFlow.swift */; }; E7D731030584957DAD52683C /* Deferred.swift in Sources */ = {isa = PBXBuildFile; fileRef = 10CFCE63856A801FB14A0633 /* Deferred.swift */; }; E8293787CB5E5CECE38A63B2 /* Encryption.swift in Sources */ = {isa = PBXBuildFile; fileRef = 54699BC0E00F327E67908F6A /* Encryption.swift */; }; @@ -293,6 +306,7 @@ FBA2EFCD6258B97659EDE5BC /* GeneratedCoverService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 925CDE3176715EBEBF40B21F /* GeneratedCoverService.swift */; }; FCC1E4CA5DE12AFBB80A3C37 /* URITemplate.swift in Sources */ = {isa = PBXBuildFile; fileRef = D5A6F75A226DE424A0515AC3 /* URITemplate.swift */; }; FD13DEAC62A3ED6714841B7A /* HTTPRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7214B2366A4E024517FF8C76 /* HTTPRequest.swift */; }; + FD1468D898D5B4CEE52378F2 /* TTSEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = D88E58FF0AC7D506273FD8D9 /* TTSEngine.swift */; }; FD16EA6468E99FB52ED97A5D /* PDFOutlineNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5D7B566F794F356878AE8E0 /* PDFOutlineNode.swift */; }; FD80A1458442254E194888F4 /* ZIPInputStream.swift in Sources */ = {isa = PBXBuildFile; fileRef = B0276C0D645E8013EE0F86FA /* ZIPInputStream.swift */; }; FE690C9C116731D017E7DB43 /* ContentProtectionService+WS.swift in Sources */ = {isa = PBXBuildFile; fileRef = 33C422C1CFB72372FC343AE4 /* ContentProtectionService+WS.swift */; }; @@ -338,6 +352,7 @@ 01D191FF1BE0BA97581EB070 /* ReadiumLicenseContainer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ReadiumLicenseContainer.swift; sourceTree = ""; }; 03C234075C7F7573BA54B77D /* EPUBParser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EPUBParser.swift; sourceTree = ""; }; 049EDB4F925E0AFEDA7318A5 /* HTTPFetcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HTTPFetcher.swift; sourceTree = ""; }; + 060213D1B559927504AFF9AF /* ContentTokenizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentTokenizer.swift; sourceTree = ""; }; 067E58BE65BCB4F8D1E8B911 /* Loggable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Loggable.swift; sourceTree = ""; }; 07B5469E40752E598C070E5B /* OPDSParser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OPDSParser.swift; sourceTree = ""; }; 093629E752DE17264B97C598 /* LCPLicense.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LCPLicense.swift; sourceTree = ""; }; @@ -346,6 +361,7 @@ 0CB0D3EE83AE0CE1F0B0B0CF /* CancellableResult.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CancellableResult.swift; sourceTree = ""; }; 0E1D7FA19C628EA8F967F580 /* Deprecated.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Deprecated.swift; sourceTree = ""; }; 0FC49AFB32B525AAC5BF7612 /* OPFMeta.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OPFMeta.swift; sourceTree = ""; }; + 1039900AC78465AD989D7464 /* Content.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Content.swift; sourceTree = ""; }; 103E0171A3CDEFA1B1F1F180 /* WKWebView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WKWebView.swift; sourceTree = ""; }; 10CFCE63856A801FB14A0633 /* Deferred.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Deferred.swift; sourceTree = ""; }; 10FB29EDCCE5910C869295F1 /* Either.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Either.swift; sourceTree = ""; }; @@ -354,6 +370,7 @@ 125BAF5FDFA097BA5CC63539 /* StringExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StringExtension.swift; sourceTree = ""; }; 15980B67505AAF10642B56C8 /* LicenseContainer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LicenseContainer.swift; sourceTree = ""; }; 17D22986A3ADE9E883691EE2 /* Deferred.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Deferred.swift; sourceTree = ""; }; + 18E809378D79D09192A0AAE1 /* ContentService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentService.swift; sourceTree = ""; }; 194C08173CDF8E3FE15D8D4A /* Collection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Collection.swift; sourceTree = ""; }; 1BE032F34E5529E3F5FD62F1 /* MediaTypeSnifferContent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaTypeSnifferContent.swift; sourceTree = ""; }; 1C22408FE1FA81400DE8D5F7 /* OPDSPrice.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OPDSPrice.swift; sourceTree = ""; }; @@ -376,10 +393,12 @@ 2DE48021CF3FED1C3340E458 /* RoutingFetcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RoutingFetcher.swift; sourceTree = ""; }; 2DF03272C07D6951ADC1311E /* Publication.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Publication.swift; sourceTree = ""; }; 300E15AA6D30BBFB7416AC01 /* MediaTypeSnifferContext.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaTypeSnifferContext.swift; sourceTree = ""; }; + 3231F989F7D7E560DD5364B9 /* Range.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Range.swift; sourceTree = ""; }; 339637CCF01E665F4CB78B01 /* EPUBLayout.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EPUBLayout.swift; sourceTree = ""; }; 33C422C1CFB72372FC343AE4 /* ContentProtectionService+WS.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ContentProtectionService+WS.swift"; sourceTree = ""; }; 33FD18E1CF87271DA6A6A783 /* Connection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Connection.swift; sourceTree = ""; }; 342D5C0FEE79A2ABEE24A43E /* CoreServices.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreServices.framework; path = System/Library/Frameworks/CoreServices.framework; sourceTree = SDKROOT; }; + 34AB954525AC159166C96A36 /* HTMLResourceContentIterator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HTMLResourceContentIterator.swift; sourceTree = ""; }; 34B5C938E4973406F110F2E6 /* OPDS1Parser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OPDS1Parser.swift; sourceTree = ""; }; 34CA9A244D941CB63515EDDE /* HREF.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HREF.swift; sourceTree = ""; }; 3510E7E84A5361BCECC90569 /* WarningLogger.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WarningLogger.swift; sourceTree = ""; }; @@ -412,6 +431,7 @@ 4944D2DB99CC59F945FDA2CA /* Bundle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Bundle.swift; sourceTree = ""; }; 49C8CE772EF8EF683D0DEE57 /* MediaType+Deprecated.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MediaType+Deprecated.swift"; sourceTree = ""; }; 4BB5D42EEF0083D833E2A572 /* Publication+OPDS.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Publication+OPDS.swift"; sourceTree = ""; }; + 4BF38F71FDEC1920325B62D3 /* PublicationContentIterator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PublicationContentIterator.swift; sourceTree = ""; }; 4E564AE6D5137499C81FEBE2 /* TargetAction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TargetAction.swift; sourceTree = ""; }; 500E55D9CA753D6D6AA76D10 /* EPUBLicenseContainer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EPUBLicenseContainer.swift; sourceTree = ""; }; 505BF8A630F7C7B96754E333 /* InMemoryPositionsService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InMemoryPositionsService.swift; sourceTree = ""; }; @@ -444,6 +464,7 @@ 6770362D551A8616EB41CBF1 /* DefaultHTTPClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DefaultHTTPClient.swift; sourceTree = ""; }; 67DEBFCD9D71243C4ACC3A49 /* LCPService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LCPService.swift; sourceTree = ""; }; 68719C5F09F9193E378DF585 /* LCPDecryptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LCPDecryptor.swift; sourceTree = ""; }; + 68FF131876FA3A63025F2662 /* Language.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Language.swift; sourceTree = ""; }; 691C96D23D42A0C6AC03B1AE /* FileAsset.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileAsset.swift; sourceTree = ""; }; 6BC71BAFF7A20D7903E6EE4D /* Properties+EPUB.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Properties+EPUB.swift"; sourceTree = ""; }; 707D6D09349FB31406847ABE /* UserProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserProperties.swift; sourceTree = ""; }; @@ -453,6 +474,7 @@ 733C1DF0A4612D888376358B /* URL.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = URL.swift; sourceTree = ""; }; 74F646B746EB27124F9456F8 /* ReadingProgression.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ReadingProgression.swift; sourceTree = ""; }; 75DFA22C741A09C81E23D084 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/LCPDialogViewController.xib; sourceTree = ""; }; + 761D7DFCF307078B7283A14E /* TextTokenizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TextTokenizer.swift; sourceTree = ""; }; 76638D3D1220E4C2620B9A80 /* Properties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Properties.swift; sourceTree = ""; }; 76E46B10FD5B26A2F41718E0 /* EPUBMetadataParser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EPUBMetadataParser.swift; sourceTree = ""; }; 77392C999C0EFF83C8F2A47F /* LCPDialogAuthentication.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LCPDialogAuthentication.swift; sourceTree = ""; }; @@ -475,6 +497,7 @@ 8B6A5B12925813FB40C41034 /* Presentation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Presentation.swift; sourceTree = ""; }; 8C0B4302E87880979A441710 /* Publication+JSON.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Publication+JSON.swift"; sourceTree = ""; }; 8D187A577EBFCFF738D1CDC7 /* ZIPFoundation.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = ZIPFoundation.xcframework; path = ../../Carthage/Build/ZIPFoundation.xcframework; sourceTree = ""; }; + 8DA31089FCAD8DFB9AC46E4E /* Tokenizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Tokenizer.swift; sourceTree = ""; }; 90AE9BB78C8A3FA5708F6AE6 /* Resource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Resource.swift; sourceTree = ""; }; 925CDE3176715EBEBF40B21F /* GeneratedCoverService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GeneratedCoverService.swift; sourceTree = ""; }; 93BF3947EBA8736BF20F36FB /* WebView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebView.swift; sourceTree = ""; }; @@ -486,6 +509,7 @@ 98D8CC7BC117BBFB206D01CC /* EPUBSpread.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EPUBSpread.swift; sourceTree = ""; }; 9935832F8ECA0AB7A7A486FC /* OPDS2Parser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OPDS2Parser.swift; sourceTree = ""; }; 999F16769EC3127CE292B8DB /* PDFDocumentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PDFDocumentView.swift; sourceTree = ""; }; + 99DE4955327D8C2DE6F866D3 /* PublicationSpeechSynthesizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PublicationSpeechSynthesizer.swift; sourceTree = ""; }; 9B5B029CA09EE1F86A19612A /* SearchService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchService.swift; sourceTree = ""; }; 9BD31F314E7B3A61C55635E5 /* prod-license.lcpl */ = {isa = PBXFileReference; path = "prod-license.lcpl"; sourceTree = ""; }; 9D586820910099E82E7C35B5 /* UserSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserSettings.swift; sourceTree = ""; }; @@ -510,12 +534,14 @@ A94DA04D56753CC008F65B1A /* VisualNavigator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VisualNavigator.swift; sourceTree = ""; }; AB0EF21FADD12D51D0619C0D /* LinkRelation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LinkRelation.swift; sourceTree = ""; }; AB1F7BC3EC3419CB824E3A70 /* ProxyFetcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProxyFetcher.swift; sourceTree = ""; }; + AB3E08C8187DCC3099CF9D22 /* Range.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Range.swift; sourceTree = ""; }; ABAF1D0444B94E2CDD80087D /* PDFKit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PDFKit.swift; sourceTree = ""; }; ACB32E55E1F3CAF1737979CC /* DataCompression.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataCompression.swift; sourceTree = ""; }; ADDB8B9906FC78C038203BDD /* User.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = User.swift; sourceTree = ""; }; AE0F9F65A46A9D2B4AF1A0FE /* BufferedResource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BufferedResource.swift; sourceTree = ""; }; B0276C0D645E8013EE0F86FA /* ZIPInputStream.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ZIPInputStream.swift; sourceTree = ""; }; B1085F2D690A73984E675D54 /* ParseData.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ParseData.swift; sourceTree = ""; }; + B15C9123EA383ED81DE0393A /* AVTTSEngine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AVTTSEngine.swift; sourceTree = ""; }; B15EC41FF314ABF15AB25CAC /* DeviceRepository.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeviceRepository.swift; sourceTree = ""; }; B2C9762191DAD823E7C925A5 /* DataExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataExtension.swift; sourceTree = ""; }; B421601FB56132514CCD9699 /* Fuzi.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = Fuzi.xcframework; path = ../../Carthage/Build/Fuzi.xcframework; sourceTree = ""; }; @@ -532,6 +558,7 @@ C084C255A327387F36B97A62 /* DOMRange.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DOMRange.swift; sourceTree = ""; }; C13A00D67725D378EB9E386C /* R2Navigator.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = R2Navigator.framework; sourceTree = BUILT_PRODUCTS_DIR; }; C2C93C33347DC0A41FE15AC6 /* License.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = License.swift; sourceTree = ""; }; + C361F965E7A7962CA3E4C0BA /* CursorList.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CursorList.swift; sourceTree = ""; }; C38A7D45005927987BFEA228 /* SMILParser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SMILParser.swift; sourceTree = ""; }; C4C94659A8749299DBE3628D /* HTTPClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HTTPClient.swift; sourceTree = ""; }; C59803AADFCF32C93C9D9D29 /* ExplodedArchive.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExplodedArchive.swift; sourceTree = ""; }; @@ -554,6 +581,7 @@ D6BCDFDD5327AB802F0F6460 /* NowPlayingInfo.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NowPlayingInfo.swift; sourceTree = ""; }; D6C93236E313B55D8B835D9F /* EPUBPositionsService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EPUBPositionsService.swift; sourceTree = ""; }; D81A35A8B299AD4B74915291 /* Fetcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Fetcher.swift; sourceTree = ""; }; + D88E58FF0AC7D506273FD8D9 /* TTSEngine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TTSEngine.swift; sourceTree = ""; }; D92391897F01AC5AFD509B1D /* GCDWebServer.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = GCDWebServer.xcframework; path = ../../Carthage/Build/GCDWebServer.xcframework; sourceTree = ""; }; D93B0556DAAAF429893B0692 /* CRLService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CRLService.swift; sourceTree = ""; }; D94EB44EC5A15FF631AE8B2E /* Rights.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Rights.swift; sourceTree = ""; }; @@ -739,6 +767,7 @@ children = ( FCEE6DBDF8E3D1ABE990DB33 /* Bundle.swift */, 48856E9AB402E2907B5230F3 /* CGRect.swift */, + AB3E08C8187DCC3099CF9D22 /* Range.swift */, 5E788FD34BE635B4B80C18A6 /* UIColor.swift */, 55D0EAD1ABB7B829A3891D3A /* UIView.swift */, 103E0171A3CDEFA1B1F1F180 /* WKWebView.swift */, @@ -918,11 +947,23 @@ path = Toolkit; sourceTree = ""; }; + 47FCB3FC286AD1053854444A /* Content */ = { + isa = PBXGroup; + children = ( + 1039900AC78465AD989D7464 /* Content.swift */, + 18E809378D79D09192A0AAE1 /* ContentService.swift */, + 060213D1B559927504AFF9AF /* ContentTokenizer.swift */, + 6D06DC650D45E72A37A19E25 /* Iterators */, + ); + path = Content; + sourceTree = ""; + }; 4898F65BFF048F7966C82B74 /* Services */ = { isa = PBXGroup; children = ( 667B76C4766DFF58D066D40B /* PublicationService.swift */, 7768FC212BAC1669A5ED08C5 /* PublicationServicesBuilder.swift */, + 47FCB3FC286AD1053854444A /* Content */, A4A409DF92515874F2F0DF6B /* Content Protection */, 3723879A352B0300CCC0006E /* Cover */, 3118D7E15D685347720A0651 /* Locator */, @@ -940,6 +981,7 @@ 194C08173CDF8E3FE15D8D4A /* Collection.swift */, C7931CB2A5658CAAECD150B0 /* NSRegularExpression.swift */, CC925E451D875E5F74748EDC /* Optional.swift */, + 3231F989F7D7E560DD5364B9 /* Range.swift */, 634444C3FD707BD99E337CDC /* Result.swift */, 57074892837A37E3BFEDB481 /* String.swift */, BB11EA964FBB42D44C3E4A50 /* StringEncoding.swift */, @@ -962,6 +1004,15 @@ path = Streams; sourceTree = ""; }; + 55198591F887417680938950 /* Tokenizer */ = { + isa = PBXGroup; + children = ( + 761D7DFCF307078B7283A14E /* TextTokenizer.swift */, + 8DA31089FCAD8DFB9AC46E4E /* Tokenizer.swift */, + ); + path = Tokenizer; + sourceTree = ""; + }; 5532F6CE3C677EB3F9B857D6 /* Model */ = { isa = PBXGroup; children = ( @@ -1054,6 +1105,15 @@ path = ../../Sources/OPDS; sourceTree = ""; }; + 6D06DC650D45E72A37A19E25 /* Iterators */ = { + isa = PBXGroup; + children = ( + 34AB954525AC159166C96A36 /* HTMLResourceContentIterator.swift */, + 4BF38F71FDEC1920325B62D3 /* PublicationContentIterator.swift */, + ); + path = Iterators; + sourceTree = ""; + }; 6D6ED6A7FC09537109EB01BF /* HTTP */ = { isa = PBXGroup; children = ( @@ -1079,6 +1139,7 @@ isa = PBXGroup; children = ( 65C8719E9CC8EF0D2430AD85 /* CompletionList.swift */, + C361F965E7A7962CA3E4C0BA /* CursorList.swift */, 9EA3A43B7709F7539F9410CD /* PaginationView.swift */, E0136BC8AC2E0F3171763FEB /* R2NavigatorLocalizedString.swift */, 4E564AE6D5137499C81FEBE2 /* TargetAction.swift */, @@ -1256,6 +1317,16 @@ name = Frameworks; sourceTree = ""; }; + BCC039E3D9F05F3D89FF5D71 /* TTS */ = { + isa = PBXGroup; + children = ( + B15C9123EA383ED81DE0393A /* AVTTSEngine.swift */, + 99DE4955327D8C2DE6F866D3 /* PublicationSpeechSynthesizer.swift */, + D88E58FF0AC7D506273FD8D9 /* TTSEngine.swift */, + ); + path = TTS; + sourceTree = ""; + }; BDC4852234E517B6C18397E2 /* Media Type */ = { isa = PBXGroup; children = ( @@ -1300,6 +1371,7 @@ 10FB29EDCCE5910C869295F1 /* Either.swift */, 34CA9A244D941CB63515EDDE /* HREF.swift */, EDA827FC94F5CB3F9032028F /* JSON.swift */, + 68FF131876FA3A63025F2662 /* Language.swift */, 5BC6AE42A31D77B548CB0BB4 /* Observable.swift */, 972DC46120E457918E69EBD0 /* R2LocalizedString.swift */, E8D7AF06866C53D07E094337 /* ResourcesServer.swift */, @@ -1313,6 +1385,7 @@ A9CBB09E0B9D74FC0D4F8A19 /* Media */, BDC4852234E517B6C18397E2 /* Media Type */, 5B825E49F38CA674DAD208D6 /* PDF */, + 55198591F887417680938950 /* Tokenizer */, 7392F4972991E267A1561E30 /* XML */, ); path = Toolkit; @@ -1388,6 +1461,7 @@ 08D09A44D576111182909F09 /* PDF */, 7F01FB1E5DDEA0BA0A04EA49 /* Resources */, 7DFC8FFCF762A897AC53DDAF /* Toolkit */, + BCC039E3D9F05F3D89FF5D71 /* TTS */, ); name = Navigator; path = ../../Sources/Navigator; @@ -1802,11 +1876,13 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 1399283B7E9E39AADA4EE7DD /* AVTTSEngine.swift in Sources */, AA0CDCC2CA63228C1F35E816 /* AudioNavigator.swift in Sources */, 99F3C8988EA41B0376D85F72 /* Bundle.swift in Sources */, 7B2C3E92CAE34EE73DDDCF10 /* CBZNavigatorViewController.swift in Sources */, 80B2146BDF073A2FF1C28426 /* CGRect.swift in Sources */, 59FD0E40847BED23C7E59FBE /* CompletionList.swift in Sources */, + 9C682824485E27814F92285F /* CursorList.swift in Sources */, 9B5F31EE78E818F890F7FBD1 /* DecorableNavigator.swift in Sources */, 98ABD996FB77EDF7DA69B18F /* DiffableDecoration+HTML.swift in Sources */, 9DF8A7CF028D764E9D6A2BAC /* DiffableDecoration.swift in Sources */, @@ -1826,8 +1902,11 @@ 70E5D945A0B0BBC84F64C173 /* PDFTapGestureController.swift in Sources */, 55AD61DD47FEE19A967EB258 /* PaginationView.swift in Sources */, B8662849CA988CD3C92D883A /* PublicationMediaLoader.swift in Sources */, + 65C6F8A05A0B3ACD2EE44944 /* PublicationSpeechSynthesizer.swift in Sources */, 5396755709F165FA1A945DEE /* R2NavigatorLocalizedString.swift in Sources */, + C4AAABD4474B6A5A25B34720 /* Range.swift in Sources */, 8E3A8F9AC2DE6F2769C1B69A /* SelectableNavigator.swift in Sources */, + FD1468D898D5B4CEE52378F2 /* TTSEngine.swift in Sources */, D248F68B569EDADA445E341D /* TargetAction.swift in Sources */, 41D9812679A98F44DA9E7BFD /* UIColor.swift in Sources */, BC959180C51A5E484D328D47 /* UIView.swift in Sources */, @@ -1854,10 +1933,13 @@ 4203767BBAACC7B330623F62 /* Cancellable.swift in Sources */, B676C73C834E530E5C019F66 /* CancellableResult.swift in Sources */, 7BDC9F1051BDD3BC61D86B09 /* Collection.swift in Sources */, + E6B6841AFFF9EFECAEE77ECC /* Content.swift in Sources */, 82BAA3EB081DD29A928958AC /* ContentLayout.swift in Sources */, 23C3C4AFA2177CED08E1B39A /* ContentProtection.swift in Sources */, FE690C9C116731D017E7DB43 /* ContentProtectionService+WS.swift in Sources */, 4C9EACE2732D23C37E627313 /* ContentProtectionService.swift in Sources */, + 812ED3E1480A1D7AA6149F69 /* ContentService.swift in Sources */, + 95DBB33898FFA425A5913F0C /* ContentTokenizer.swift in Sources */, 2F7730648C4FA4A921038A7F /* Contributor.swift in Sources */, E6BF3A99E6C6AAC4FEE1099F /* ControlFlow.swift in Sources */, C657A9D08F53A44658962E83 /* CoverService.swift in Sources */, @@ -1884,6 +1966,7 @@ FBA2EFCD6258B97659EDE5BC /* GeneratedCoverService.swift in Sources */, 8DACB70852CEA8D64F8BEDB1 /* Group.swift in Sources */, 9BD8989B1CADB1712B31E0A4 /* HREF.swift in Sources */, + 198089C002038C10FDFBA2BF /* HTMLResourceContentIterator.swift in Sources */, 8EE4317BE92998698D48EF72 /* HTTPClient.swift in Sources */, 08234B61E941DD78EB24485B /* HTTPError.swift in Sources */, 39FC65D3797EF5069A04F34B /* HTTPFetcher.swift in Sources */, @@ -1891,6 +1974,7 @@ FD13DEAC62A3ED6714841B7A /* HTTPRequest.swift in Sources */, EA8C7F894E3BE8D6D954DC47 /* InMemoryPositionsService.swift in Sources */, 9C6B7AFB6FB0635EF5B7B71C /* JSON.swift in Sources */, + 411B624A5AE5189875950DDA /* Language.swift in Sources */, 69150D0B00F5665C3DA0000B /* LazyResource.swift in Sources */, 5C9617AE1B5678A95ABFF1AA /* Link.swift in Sources */, C784A3821288A580700AD1DB /* LinkRelation.swift in Sources */, @@ -1943,9 +2027,11 @@ 134AF2657ABA617255DE2D0A /* Publication.swift in Sources */, 8B8A6E58C84597087280BA20 /* PublicationAsset.swift in Sources */, 037E68E96839B96F547BDD6E /* PublicationCollection.swift in Sources */, + C35001848411CBCAC8F03763 /* PublicationContentIterator.swift in Sources */, 88A171A36700ACF5A4AD6305 /* PublicationService.swift in Sources */, CC1EBC553CE8A5C873E7A9BB /* PublicationServicesBuilder.swift in Sources */, CA152829D0654EB38D6BF836 /* R2LocalizedString.swift in Sources */, + 501E7E05DEA11F7A61D60EAF /* Range.swift in Sources */, 2D65E93D77922E33DA03D638 /* ReadingProgression.swift in Sources */, 3B1820FD0226743B1DE41FCF /* Resource.swift in Sources */, 95B9369AE4743FB7BAB93DCC /* ResourceContentExtractor.swift in Sources */, @@ -1958,6 +2044,8 @@ B5DC9710E7124907BBFE9EA5 /* StringEncoding.swift in Sources */, FFC0D2E981B9AB2246831B56 /* StringSearchService.swift in Sources */, 2BD38736DB1971926FA77234 /* Subject.swift in Sources */, + 99856B9FCC56A9F1946C6A60 /* TextTokenizer.swift in Sources */, + CD7DF8DC7B346AA86DECE596 /* Tokenizer.swift in Sources */, 0B3F1407E77E6825F66849DA /* TransformingFetcher.swift in Sources */, 718323B1A0C981D1B7A08F91 /* TransformingResource.swift in Sources */, 52C4CB868EA5FBFBB43DD65C /* UIImage.swift in Sources */, From 68c8996766e304ae581af4d8e2042290d2d41fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Sat, 13 Aug 2022 15:43:45 +0200 Subject: [PATCH 45/46] Fix build on older Xcode versions --- Sources/Shared/Publication/Locator.swift | 4 ++-- .../Publication/Services/Content/ContentTokenizer.swift | 4 ++-- .../Content/Iterators/HTMLResourceContentIterator.swift | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/Shared/Publication/Locator.swift b/Sources/Shared/Publication/Locator.swift index 64085ee84..cb677ccef 100644 --- a/Sources/Shared/Publication/Locator.swift +++ b/Sources/Shared/Publication/Locator.swift @@ -278,8 +278,8 @@ public struct Locator: Hashable, CustomStringConvertible, Loggable { } let range = range.relative(to: highlight) - var before = before ?? "" - var after = after ?? "" + var before = self.before ?? "" + var after = self.after ?? "" let newHighlight = highlight[range] before = before + highlight[.. TextTokenizer ) -> ContentTokenizer { - func tokenize(_ segment: TextContentElement.Segment) throws -> [TextContentElement.Segment] { + func tokenize(segment: TextContentElement.Segment) throws -> [TextContentElement.Segment] { let tokenize = textTokenizerFactory(segment.language ?? defaultLanguage) return try tokenize(segment.text) @@ -37,7 +37,7 @@ public func makeTextContentTokenizer( func tokenize(_ content: ContentElement) throws -> [ContentElement] { if var content = content as? TextContentElement { - content.segments = try content.segments.flatMap(tokenize) + content.segments = try content.segments.flatMap(tokenize(segment:)) return [content] } else { return [content] diff --git a/Sources/Shared/Publication/Services/Content/Iterators/HTMLResourceContentIterator.swift b/Sources/Shared/Publication/Services/Content/Iterators/HTMLResourceContentIterator.swift index 420f56dfd..9e4a03f11 100644 --- a/Sources/Shared/Publication/Services/Content/Iterators/HTMLResourceContentIterator.swift +++ b/Sources/Shared/Publication/Services/Content/Iterators/HTMLResourceContentIterator.swift @@ -44,7 +44,7 @@ public class HTMLResourceContentIterator: ContentIterator { } private func next(by delta: Int) throws -> ContentElement? { - let elements = try elements.get() + let elements = try self.elements.get() let index = currentIndex.map { $0 + delta } ?? elements.startIndex From 669a3df0d758f4d6e4ed095c90bad4554881efb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mickae=CC=88l=20Menu?= Date: Fri, 26 Aug 2022 13:40:30 +0200 Subject: [PATCH 46/46] Fix Carthage project --- Support/Carthage/.xcodegen | 2 +- .../xcshareddata/xcschemes/R2Navigator.xcscheme | 5 ++--- .../xcshareddata/xcschemes/R2Shared.xcscheme | 5 ++--- .../xcshareddata/xcschemes/R2Streamer.xcscheme | 5 ++--- .../xcshareddata/xcschemes/ReadiumLCP.xcscheme | 5 ++--- .../xcshareddata/xcschemes/ReadiumOPDS.xcscheme | 5 ++--- 6 files changed, 11 insertions(+), 16 deletions(-) diff --git a/Support/Carthage/.xcodegen b/Support/Carthage/.xcodegen index 40bd35200..fd668ab20 100644 --- a/Support/Carthage/.xcodegen +++ b/Support/Carthage/.xcodegen @@ -1,5 +1,5 @@ # XCODEGEN VERSION -2.31.0 +2.32.0 # SPEC { diff --git a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Navigator.xcscheme b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Navigator.xcscheme index 406a8a2f3..d07c5307a 100644 --- a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Navigator.xcscheme +++ b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Navigator.xcscheme @@ -67,8 +67,7 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" debugDocumentVersioning = "YES"> - + - + diff --git a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Shared.xcscheme b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Shared.xcscheme index 964a6c38e..49199b007 100644 --- a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Shared.xcscheme +++ b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Shared.xcscheme @@ -67,8 +67,7 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" debugDocumentVersioning = "YES"> - + - + diff --git a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Streamer.xcscheme b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Streamer.xcscheme index 233c0e79b..73d71f104 100644 --- a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Streamer.xcscheme +++ b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/R2Streamer.xcscheme @@ -67,8 +67,7 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" debugDocumentVersioning = "YES"> - + - + diff --git a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/ReadiumLCP.xcscheme b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/ReadiumLCP.xcscheme index d18e9630d..8c762ff9c 100644 --- a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/ReadiumLCP.xcscheme +++ b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/ReadiumLCP.xcscheme @@ -67,8 +67,7 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" debugDocumentVersioning = "YES"> - + - + diff --git a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/ReadiumOPDS.xcscheme b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/ReadiumOPDS.xcscheme index 6fdac9f9d..149e1899b 100644 --- a/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/ReadiumOPDS.xcscheme +++ b/Support/Carthage/Readium.xcodeproj/xcshareddata/xcschemes/ReadiumOPDS.xcscheme @@ -67,8 +67,7 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" debugDocumentVersioning = "YES"> - + - +