From 80344c3bf00cbb3741915876deaedd505413def3 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 8 Oct 2021 10:07:40 +0000 Subject: [PATCH 01/45] Add endpoint `/api/login` --- src/handlers/auth_handler.cr | 5 +++-- src/routes/api.cr | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/handlers/auth_handler.cr b/src/handlers/auth_handler.cr index 692fa8a8..bf79dc38 100644 --- a/src/handlers/auth_handler.cr +++ b/src/handlers/auth_handler.cr @@ -54,8 +54,9 @@ class AuthHandler < Kemal::Handler end def call(env) - # Skip all authentication if requesting /login, /logout, or a static file - if request_path_startswith(env, ["/login", "/logout"]) || + # Skip all authentication if requesting /login, /logout, /api/login, + # or a static file + if request_path_startswith(env, ["/login", "/logout", "/api/login"]) || requesting_static_file env return call_next(env) end diff --git a/src/routes/api.cr b/src/routes/api.cr index ed9bb299..1fea598b 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -23,7 +23,7 @@ struct APIRouter # Authentication - All endpoints require authentication. After logging in, your session ID would be stored as a cookie named `mango-sessid-#{Config.current.port}`, which can be used to authenticate the API access. Note that all admin API endpoints (`/api/admin/...`) require the logged-in user to have admin access. + All endpoints except `/api/login` require authentication. After logging in, your session ID would be stored as a cookie named `mango-sessid-#{Config.current.port}`, which can be used to authenticate the API access. Note that all admin API endpoints (`/api/admin/...`) require the logged-in user to have admin access. # Terminologies @@ -56,6 +56,29 @@ struct APIRouter "error" => String?, } + Koa.describe "Authenticates a user", <<-MD + After successful login, the cookie `mango-sessid-#{Config.current.port}` will contain a valid session ID that can be used for subsequent requests + MD + Koa.body schema: { + "username" => String, + "password" => String, + } + Koa.tag "users" + post "/api/login" do |env| + begin + username = env.params.json["username"].as String + password = env.params.json["password"].as String + token = Storage.default.verify_user(username, password).not_nil! + + env.session.string "token", token + "Authenticated" + rescue e + Logger.error e + env.response.status_code = 403 + e.message + end + end + Koa.describe "Returns a page in a manga entry" Koa.path "tid", desc: "Title ID" Koa.path "eid", desc: "Entry ID" From 1199eb7a03ebbd2a7ebbd033ffc00a6e7ebe537c Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 16 Nov 2021 13:37:19 +0000 Subject: [PATCH 02/45] Use mobile menu at @m (fixes #246) --- src/views/layout.html.ecr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/views/layout.html.ecr b/src/views/layout.html.ecr index 70c5a51f..c32bfb5b 100644 --- a/src/views/layout.html.ecr +++ b/src/views/layout.html.ecr @@ -32,10 +32,10 @@
-
+
-
+
  • Home
    • @@ -57,7 +57,7 @@ <% end %>
-
+
  • Logout
    • From 921628ba6d251e5cd896c859f15a37c2f1696f5c Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Wed, 17 Nov 2021 13:10:44 +0000 Subject: [PATCH 03/45] Limit max length in download table (fixes #244) --- public/js/plugin-download.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/public/js/plugin-download.js b/public/js/plugin-download.js index a335e038..2346fe27 100644 --- a/public/js/plugin-download.js +++ b/public/js/plugin-download.js @@ -68,7 +68,12 @@ const buildTable = (chapters) => { $('table').append(thead); const rows = chapters.map(ch => { - const tds = Object.values(ch).map(v => `${v}`).join(''); + const tds = Object.values(ch).map(v => { + const maxLength = 40; + const shouldShrink = v.length > maxLength; + const content = shouldShrink ? `${v.substring(0, maxLength)}...
    ${v}
    ` : v; + return `${content}` + }).join(''); return `${tds}`; }); const tbody = `${rows}`; From cc33fa659597ff0ebc0dbaa6f007106073005c7a Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 19 Dec 2021 01:18:31 +0900 Subject: [PATCH 04/45] Fix bug: remove titles not in root library anymore --- src/library/library.cr | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 93eac0c9..2a5de7db 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -136,8 +136,12 @@ class Library deleted_entry_ids: [] of String, } + library_paths = (Dir.entries @dir) + .select { |fn| !fn.starts_with? "." } + .map { |fn| File.join @dir, fn } @title_ids.select! do |title_id| title = @title_hash[title_id] + next false unless library_paths.includes? title.dir existence = title.examine examine_context unless existence examine_context["deleted_title_ids"].concat [title_id] + @@ -152,9 +156,7 @@ class Library end cache = examine_context["cached_contents_signature"] - (Dir.entries @dir) - .select { |fn| !fn.starts_with? "." } - .map { |fn| File.join @dir, fn } + library_paths .select { |path| !(remained_title_dirs.includes? path) } .select { |path| File.directory? path } .map { |path| Title.new path, "", cache } From 0fa2bfa744be72b8e3052c990024e079ee7edab1 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 19 Dec 2021 16:40:38 +0900 Subject: [PATCH 05/45] Fix bug on examine --- src/library/title.cr | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 9b797f41..d45e4eb2 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -102,7 +102,11 @@ class Title previous_titles_size = @title_ids.size @title_ids.select! do |title_id| - title = Library.default.get_title! title_id + title = Library.default.get_title title_id + unless title # for if data consistency broken + context["deleted_title_ids"].concat [title_id] + next false + end existence = title.examine context unless existence context["deleted_title_ids"].concat [title_id] + @@ -137,6 +141,18 @@ class Title Library.default.title_hash[title.id] = title @title_ids << title.id is_titles_added = true + + # We think they are removed, but they are here! + # Cancel reserved jobs + revival_title_ids = [title.id] + title.deep_titles.map &.id + context["deleted_title_ids"].select! do |id| + !(revival_title_ids.includes? id) + end + revival_entry_ids = title.deep_entries.map &.id + context["deleted_entry_ids"].select! do |id| + !(revival_entry_ids.includes? id) + end + next end if is_supported_file path @@ -167,7 +183,12 @@ class Title end end - true + if @title_ids.size > 0 || @entries.size > 0 + true + else + context["deleted_title_ids"].concat [@id] + false + end end alias SortContext = NamedTuple(username: String, opt: SortOptions) From 70259d8e502614f71fc365a2873964d512fa735e Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 19 Dec 2021 17:03:10 +0900 Subject: [PATCH 06/45] Do same with an entry --- src/library/title.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/library/title.cr b/src/library/title.cr index d45e4eb2..24c4841b 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -161,6 +161,7 @@ class Title if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true + context["deleted_entry_ids"].select! { |id| entry.id != id } end end end From 6d834e9164ed4b20a24e27a575189a9427c47014 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 19 Dec 2021 17:03:39 +0900 Subject: [PATCH 07/45] Fix formatting --- src/routes/admin.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/admin.cr b/src/routes/admin.cr index fd63ec82..6ff36db5 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -61,7 +61,7 @@ struct AdminRouter redirect_url = URI.new \ path: "/admin/user/edit", query: hash_to_query({"username" => original_username, \ - "admin" => admin, "error" => e.message}) + "admin" => admin, "error" => e.message}) redirect env, redirect_url.to_s end From f9254c49a10fafcbb9e8be794bc33ffd8ca60dce Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 19 Dec 2021 17:14:47 +0900 Subject: [PATCH 08/45] Fix lint error --- src/library/title.cr | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 24c4841b..4886f3d8 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -145,12 +145,12 @@ class Title # We think they are removed, but they are here! # Cancel reserved jobs revival_title_ids = [title.id] + title.deep_titles.map &.id - context["deleted_title_ids"].select! do |id| - !(revival_title_ids.includes? id) + context["deleted_title_ids"].select! do |deleted_title_id| + !(revival_title_ids.includes? deleted_title_id) end revival_entry_ids = title.deep_entries.map &.id - context["deleted_entry_ids"].select! do |id| - !(revival_entry_ids.includes? id) + context["deleted_entry_ids"].select! do |deleted_entry_id| + !(revival_entry_ids.includes? deleted_entry_id) end next @@ -161,7 +161,9 @@ class Title if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true - context["deleted_entry_ids"].select! { |id| entry.id != id } + context["deleted_entry_ids"].select! do |deleted_entry_id| + entry.id != deleted_entry_id + end end end end From 4eda55552b5506417c5d6927b604ce92d2c7f3be Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Wed, 22 Dec 2021 12:16:44 +0000 Subject: [PATCH 09/45] Linter fix --- src/routes/admin.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 6ff36db5..fd63ec82 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -61,7 +61,7 @@ struct AdminRouter redirect_url = URI.new \ path: "/admin/user/edit", query: hash_to_query({"username" => original_username, \ - "admin" => admin, "error" => e.message}) + "admin" => admin, "error" => e.message}) redirect env, redirect_url.to_s end From 57683d1cfbea75bb3f95da3653ca40ea5bea5edb Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 24 Dec 2021 16:29:55 +0900 Subject: [PATCH 10/45] Add sort option "Name" for title --- src/util/util.cr | 2 ++ src/views/library.html.ecr | 1 + 2 files changed, 3 insertions(+) diff --git a/src/util/util.cr b/src/util/util.cr index 9f5ffee6..7278bb25 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -98,6 +98,8 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String) (a.load_percentage(username) <=> b.load_percentage(username)).or \ compare_numerically a.title, b.title end + when .title? + ary.sort! { |a, b| compare_numerically a.title, b.title } else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ diff --git a/src/views/library.html.ecr b/src/views/library.html.ecr index 39e98565..a5e8b59e 100644 --- a/src/views/library.html.ecr +++ b/src/views/library.html.ecr @@ -10,6 +10,7 @@
    <% hash = { "auto" => "Auto", + "title" => "Name", "time_modified" => "Date Modified", "progress" => "Progress" } %> From 3a932d7b0a1e3213cb009c3abfde38bc25593b8a Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 24 Dec 2021 22:57:11 +0900 Subject: [PATCH 11/45] Add column 'sort_title' to titles, ids table --- migration/sort_title.12.cr | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 migration/sort_title.12.cr diff --git a/migration/sort_title.12.cr b/migration/sort_title.12.cr new file mode 100644 index 00000000..8da828fb --- /dev/null +++ b/migration/sort_title.12.cr @@ -0,0 +1,17 @@ +class SortTitle < MG::Base + def up : String + <<-SQL + -- add sort_title column to ids and titles + ALTER TABLE ids ADD COLUMN sort_title TEXT; + ALTER TABLE titles ADD COLUMN sort_title TEXT; + SQL + end + + def down : String + <<-SQL + -- drop sort_title column to ids and titles + ALTER TABLE ids DROP COLUMN sort_title; + ALTER TABLE titles DROP COLUMN sort_title; + SQL + end +end From ab799af866ad6ed80664e6bdabaf64ad41e29b71 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 24 Dec 2021 23:48:52 +0900 Subject: [PATCH 12/45] Implement sort_title getter, setter --- src/library/entry.cr | 24 ++++++++++++++++++++++++ src/library/title.cr | 23 +++++++++++++++++++++++ src/storage.cr | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/src/library/entry.cr b/src/library/entry.cr index 43fbb230..9e676c84 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -8,6 +8,9 @@ class Entry size : String, pages : Int32, id : String, encoded_path : String, encoded_title : String, mtime : Time, err_msg : String? + @[YAML::Field(ignore: true)] + @sort_title : String? + def initialize(@zip_path, @book) storage = Storage.default @encoded_path = URI.encode @zip_path @@ -66,6 +69,27 @@ class Entry end end + def sort_title + sort_title_cached = @sort_title + return sort_title_cached if sort_title_cached + sort_title = Storage.default.get_entry_sort_title id + if sort_title + @sort_title = sort_title + return sort_title + end + @sort_title = @title + @title + end + + def sort_title=(sort_title : String | Nil) + Storage.default.set_entry_sort_title id, sort_title + @sort_title = sort_title + end + + def sort_title_db + Storage.default.get_entry_sort_title id + end + def display_name @book.display_name @title end diff --git a/src/library/title.cr b/src/library/title.cr index 4886f3d8..d2e34a2d 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -10,6 +10,8 @@ class Title entry_cover_url_cache : Hash(String, String)? setter entry_cover_url_cache : Hash(String, String)? + @[YAML::Field(ignore: true)] + @sort_title : String? @[YAML::Field(ignore: true)] @entry_display_name_cache : Hash(String, String)? @[YAML::Field(ignore: true)] @@ -286,6 +288,27 @@ class Title ary.join " and " end + def sort_title + sort_title_cached = @sort_title + return sort_title_cached if sort_title_cached + sort_title = Storage.default.get_title_sort_title id + if sort_title + @sort_title = sort_title + return sort_title + end + @sort_title = @title + @title + end + + def sort_title=(sort_title : String | Nil) + Storage.default.set_title_sort_title id, sort_title + @sort_title = sort_title + end + + def sort_title_db + Storage.default.get_title_sort_title id + end + def tags Storage.default.get_title_tags @id end diff --git a/src/storage.cr b/src/storage.cr index 32f446a3..edf71ff4 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -342,6 +342,44 @@ class Storage end end + def get_title_sort_title(title_id : String) + sort_title = nil + MainFiber.run do + get_db do |db| + sort_title = db.query_one? "Select sort_title from titles where id = (?)", title_id, as: String | Nil + end + end + sort_title + end + + def set_title_sort_title(title_id : String, sort_title : String | Nil) + sort_title = nil if sort_title == "" + MainFiber.run do + get_db do |db| + db.exec "update titles set sort_title = (?) where id = (?)", sort_title, title_id + end + end + end + + def get_entry_sort_title(entry_id : String) + sort_title = nil + MainFiber.run do + get_db do |db| + sort_title = db.query_one? "Select sort_title from ids where id = (?)", entry_id, as: String | Nil + end + end + sort_title + end + + def set_entry_sort_title(entry_id : String, sort_title : String | Nil) + sort_title = nil if sort_title == "" + MainFiber.run do + get_db do |db| + db.exec "update ids set sort_title = (?) where id = (?)", sort_title, entry_id + end + end + end + def save_thumbnail(id : String, img : Image) MainFiber.run do get_db do |db| From 546bd0138cbacb31a32a51509efbded46d535aae Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 25 Dec 2021 16:50:00 +0900 Subject: [PATCH 13/45] Use sort_title instead of title --- src/library/library.cr | 2 +- src/library/title.cr | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 2a5de7db..a2dc28d9 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -161,7 +161,7 @@ class Library .select { |path| File.directory? path } .map { |path| Title.new path, "", cache } .select { |title| !(title.entries.empty? && title.titles.empty?) } - .sort! { |a, b| a.title <=> b.title } + .sort! { |a, b| a.sort_title <=> b.sort_title } .each do |title| @title_hash[title.id] = title @title_ids << title.id diff --git a/src/library/title.cr b/src/library/title.cr index d2e34a2d..e264ae9b 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -68,7 +68,7 @@ class Title end sorter = ChapterSorter.new @entries.map &.title @entries.sort! do |a, b| - sorter.compare a.title, b.title + sorter.compare a.sort_title, b.sort_title end end @@ -182,9 +182,9 @@ class Title end end if is_entries_added || previous_entries_size != @entries.size - sorter = ChapterSorter.new @entries.map &.title + sorter = ChapterSorter.new @entries.map &.sort_title @entries.sort! do |a, b| - sorter.compare a.title, b.title + sorter.compare a.sort_title, b.sort_title end end @@ -495,28 +495,28 @@ class Title case opt.not_nil!.method when .title? - ary = @entries.sort { |a, b| compare_numerically a.title, b.title } + ary = @entries.sort { |a, b| compare_numerically a.sort_title, b.sort_title } when .time_modified? ary = @entries.sort { |a, b| (a.mtime <=> b.mtime).or \ - compare_numerically a.title, b.title } + compare_numerically a.sort_title, b.sort_title } when .time_added? ary = @entries.sort { |a, b| (a.date_added <=> b.date_added).or \ - compare_numerically a.title, b.title } + compare_numerically a.sort_title, b.sort_title } when .progress? percentage_ary = load_percentage_for_all_entries username, opt, true ary = @entries.zip(percentage_ary) .sort { |a_tp, b_tp| (a_tp[1] <=> b_tp[1]).or \ - compare_numerically a_tp[0].title, b_tp[0].title } + compare_numerically a_tp[0].sort_title, b_tp[0].sort_title } .map &.[0] else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ "Auto instead" end - sorter = ChapterSorter.new @entries.map &.title + sorter = ChapterSorter.new @entries.map &.sort_title ary = @entries.sort do |a, b| - sorter.compare(a.title, b.title).or \ - compare_numerically a.title, b.title + sorter.compare(a.sort_title, b.sort_title).or \ + compare_numerically a.sort_title, b.sort_title end end From d4b58e91d10a171744e0ab41a19d12705a2f0a52 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 25 Dec 2021 16:45:32 +0900 Subject: [PATCH 14/45] Implement sort title api --- src/routes/api.cr | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/routes/api.cr b/src/routes/api.cr index 1fea598b..95d29ee3 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -371,6 +371,37 @@ struct APIRouter end end + Koa.describe "Sets the sort title of a title or an entry", <<-MD + When `eid` is provided, apply the sort title to the entry. Otherwise, apply the sort title to the title identified by `tid`. + MD + Koa.tags ["admin", "library"] + Koa.path "tid", desc: "Title ID" + Koa.query "eid", desc: "Entry ID", required: false + Koa.query "name", desc: "The new sort title" + Koa.response 200, schema: "result" + put "/api/admin/sort_title/:tid" do |env| + begin + title = (Library.default.get_title env.params.url["tid"]) + .not_nil! + name = env.params.query["name"]? + entry = env.params.query["eid"]? + if entry.nil? + title.sort_title = name + else + eobj = title.get_entry entry + eobj.sort_title = name unless eobj.nil? + end + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + else + send_json env, {"success" => true}.to_json + end + end + ws "/api/admin/mangadex/queue" do |socket, env| interval_raw = env.params.query["interval"]? interval = (interval_raw.to_i? if interval_raw) || 5 From 162318cf4a34e9598f98e666aa2785700d221542 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 25 Dec 2021 20:58:09 +0900 Subject: [PATCH 15/45] Add Api call --- public/js/title.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/public/js/title.js b/public/js/title.js index 1aca6d64..32ab7bfb 100644 --- a/public/js/title.js +++ b/public/js/title.js @@ -122,6 +122,34 @@ const renameSubmit = (name, eid) => { }); }; +const renameSortNameSubmit = (name, eid) => { + const upload = $('.upload-field'); + const titleId = upload.attr('data-title-id'); + + const params = {}; + if (eid) params.eid = eid; + if (name) params.name = name; + const query = $.param(params); + let url = `${base_url}api/admin/sort_title/${titleId}?${query}`; + + $.ajax({ + type: 'PUT', + url, + contentType: 'application/json', + dataType: 'json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to update sort title. Error: ${data.error}`); + return; + } + location.reload(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to update sort title. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); +}; + const edit = (eid) => { const cover = $('#edit-modal #cover'); let url = cover.attr('data-title-cover'); From 45ffa3d428b6ef9a9571a776a0a37471400a5bb5 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 25 Dec 2021 21:02:57 +0900 Subject: [PATCH 16/45] Implement UI to edit sort title --- public/js/title.js | 17 +++++++++++++++++ src/views/components/card.html.ecr | 4 +++- src/views/title.html.ecr | 10 +++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/public/js/title.js b/public/js/title.js index 32ab7bfb..5a2da33e 100644 --- a/public/js/title.js +++ b/public/js/title.js @@ -154,11 +154,15 @@ const edit = (eid) => { const cover = $('#edit-modal #cover'); let url = cover.attr('data-title-cover'); let displayName = $('h2.uk-title > span').text(); + let fileTitle = $('h2.uk-title').attr('data-file-title'); + let sortTitle = $('h2.uk-title').attr('data-sort-title'); if (eid) { const item = $(`#${eid}`); url = item.find('img').attr('data-src'); displayName = item.find('.uk-card-title').attr('data-title'); + fileTitle = item.find('.uk-card-title').attr('data-file-title'); + sortTitle = item.find('.uk-card-title').attr('data-sort-title'); $('#title-progress-control').attr('hidden', ''); } else { $('#title-progress-control').removeAttr('hidden'); @@ -178,6 +182,19 @@ const edit = (eid) => { renameSubmit(displayNameField.val(), eid); }); + const sortTitleField = $('#sort-title-field'); + sortTitleField.attr('value', sortTitle); + sortTitleField.attr('placeholder', fileTitle); + console.log(sortTitle); + sortTitleField.keyup(event => { + if (event.keyCode === 13) { + renameSortNameSubmit(sortTitleField.val(), eid); + } + }); + sortTitleField.siblings('a.uk-form-icon').click(() => { + renameSortNameSubmit(sortTitleField.val(), eid); + }); + setupUpload(eid); UIkit.modal($('#edit-modal')).show(); diff --git a/src/views/components/card.html.ecr b/src/views/components/card.html.ecr index b85d39e8..55494992 100644 --- a/src/views/components/card.html.ecr +++ b/src/views/components/card.html.ecr @@ -61,7 +61,9 @@ <% if page == "home" && item.is_a? Entry %> <%= "uk-margin-remove-bottom" %> <% end %> - " data-title="<%= HTML.escape(item.display_name) %>"><%= HTML.escape(item.display_name) %> + " data-title="<%= HTML.escape(item.display_name) %>" + data-file-title="<%= HTML.escape(item.title || "") %>" + data-sort-title="<%= HTML.escape(item.sort_title_db || "") %>"><%= HTML.escape(item.display_name) %> <% if page == "home" && item.is_a? Entry %> <%= HTML.escape(item.book.display_name) %> diff --git a/src/views/title.html.ecr b/src/views/title.html.ecr index 78edf988..99b93803 100644 --- a/src/views/title.html.ecr +++ b/src/views/title.html.ecr @@ -18,7 +18,8 @@
-

<%= title.display_name %> +

"> + <%= title.display_name %>   <% if is_admin %> @@ -89,6 +90,13 @@

+
+ +
+ + +
+
From edfef80e5c11418c8a7310337a365519ade77de4 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 25 Dec 2021 22:12:09 +0900 Subject: [PATCH 17/45] Invalidate sort result cache after change sort_title --- src/library/entry.cr | 10 +++++++++- src/library/title.cr | 11 ++++++++++- src/routes/api.cr | 5 +++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 9e676c84..0d508f13 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -81,9 +81,17 @@ class Entry @title end - def sort_title=(sort_title : String | Nil) + def set_sort_title(sort_title : String | Nil, username : String) Storage.default.set_entry_sort_title id, sort_title @sort_title = sort_title + + [false, true].each do |ascend| + [SortMethod::Auto, SortMethod::Title].each do |sort_method| + sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id, + username, @book.entries, SortOptions.new(sort_method, ascend) + LRUCache.invalidate sorted_entries_cache_key + end + end end def sort_title_db diff --git a/src/library/title.cr b/src/library/title.cr index e264ae9b..1c897313 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -300,9 +300,18 @@ class Title @title end - def sort_title=(sort_title : String | Nil) + def set_sort_title(sort_title : String | Nil, username : String) Storage.default.set_title_sort_title id, sort_title @sort_title = sort_title + + [false, true].each do |ascend| + [SortMethod::Auto, SortMethod::Title].each do |sort_method| + sorted_entries_cache_key = + SortedEntriesCacheEntry.gen_key @id, username, @entries, + SortOptions.new(sort_method, ascend) + LRUCache.invalidate sorted_entries_cache_key + end + end end def sort_title_db diff --git a/src/routes/api.cr b/src/routes/api.cr index 95d29ee3..28ec314b 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -380,16 +380,17 @@ struct APIRouter Koa.query "name", desc: "The new sort title" Koa.response 200, schema: "result" put "/api/admin/sort_title/:tid" do |env| + username = get_username env begin title = (Library.default.get_title env.params.url["tid"]) .not_nil! name = env.params.query["name"]? entry = env.params.query["eid"]? if entry.nil? - title.sort_title = name + title.set_sort_title name, username else eobj = title.get_entry entry - eobj.sort_title = name unless eobj.nil? + eobj.set_sort_title name, username unless eobj.nil? end rescue e Logger.error e From 234b29bbdd38a42cbd462c1c275d1ec3575f1382 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 25 Dec 2021 22:40:10 +0900 Subject: [PATCH 18/45] Fix save --- src/library/entry.cr | 6 +++++- src/library/title.cr | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 0d508f13..ec11cb12 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -83,7 +83,11 @@ class Entry def set_sort_title(sort_title : String | Nil, username : String) Storage.default.set_entry_sort_title id, sort_title - @sort_title = sort_title + if sort_title == "" || sort_title.nil? + @sort_title = nil + else + @sort_title = sort_title + end [false, true].each do |ascend| [SortMethod::Auto, SortMethod::Title].each do |sort_method| diff --git a/src/library/title.cr b/src/library/title.cr index 1c897313..883f36c1 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -302,7 +302,11 @@ class Title def set_sort_title(sort_title : String | Nil, username : String) Storage.default.set_title_sort_title id, sort_title - @sort_title = sort_title + if sort_title == "" || sort_title.nil? + @sort_title = nil + else + @sort_title = sort_title + end [false, true].each do |ascend| [SortMethod::Auto, SortMethod::Title].each do |sort_method| From 8fea35fa510cc3210318046d65fc72670e3cc5cd Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 26 Dec 2021 00:27:00 +0900 Subject: [PATCH 19/45] Use sorted_titles --- src/library/title.cr | 4 ++++ src/routes/main.cr | 6 ++++++ src/views/title.html.ecr | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 883f36c1..d1520be3 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -252,6 +252,10 @@ class Title @title_ids.map { |tid| Library.default.get_title! tid } end + def sorted_titles(username, opt : SortOptions? = nil) + titles + end + # Get all entries, including entries in nested titles def deep_entries return @entries if title_ids.empty? diff --git a/src/routes/main.cr b/src/routes/main.cr index 4aa7da63..87a7c3ba 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -61,9 +61,15 @@ struct MainRouter sort_opt = SortOptions.from_info_json title.dir, username get_and_save_sort_opt title.dir + sorted_titles = title.sorted_titles username, sort_opt entries = title.sorted_entries username, sort_opt percentage = title.load_percentage_for_all_entries username, sort_opt title_percentage = title.titles.map &.load_percentage username + title_percentage_map = {} of String => Float64 + title_percentage.each_with_index do |percentage, i| + t = title.titles[i] + title_percentage_map[t.id] = percentage + end layout "title" rescue e diff --git a/src/views/title.html.ecr b/src/views/title.html.ecr index 99b93803..68803475 100644 --- a/src/views/title.html.ecr +++ b/src/views/title.html.ecr @@ -60,8 +60,8 @@
- <% title.titles.each_with_index do |item, i| %> - <% progress = title_percentage[i] %> + <% sorted_titles.each do |item| %> + <% progress = title_percentage_map[item.id] %> <%= render_component "card" %> <% end %>
From 1ec8dcbfdab19c8bf88d9781e9e22ca03b0ac242 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 26 Dec 2021 01:50:55 +0900 Subject: [PATCH 20/45] Use sort_title, sort_titles in title page --- src/library/title.cr | 7 ++++++- src/util/util.cr | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index d1520be3..9eacf4eb 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -253,7 +253,12 @@ class Title end def sorted_titles(username, opt : SortOptions? = nil) - titles + if opt.nil? + opt = SortOptions.from_info_json @dir, username + end + + # Helper function from src/util/util.cr + sort_titles titles, opt.not_nil!, username end # Get all entries, including entries in nested titles diff --git a/src/util/util.cr b/src/util/util.cr index 7278bb25..2814c6c4 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -91,21 +91,21 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String) case opt.method when .time_modified? - ary.sort! { |a, b| (a.mtime <=> b.mtime).or \ - compare_numerically a.title, b.title } + ary.sort { |a, b| (a.mtime <=> b.mtime).or \ + compare_numerically a.sort_title, b.sort_title } when .progress? - ary.sort! do |a, b| + ary.sort do |a, b| (a.load_percentage(username) <=> b.load_percentage(username)).or \ - compare_numerically a.title, b.title + compare_numerically a.sort_title, b.sort_title end when .title? - ary.sort! { |a, b| compare_numerically a.title, b.title } + ary.sort { |a, b| compare_numerically a.sort_title, b.sort_title } else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ "Auto instead" end - ary.sort! { |a, b| compare_numerically a.title, b.title } + ary.sort { |a, b| compare_numerically a.sort_title, b.sort_title } end ary.reverse! unless opt.not_nil!.ascend From 1cd777d27d08b5bbc8c7d5fa99e46c59e6240bd7 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 26 Dec 2021 02:56:57 +0900 Subject: [PATCH 21/45] Cache sorted titles --- src/library/cache.cr | 31 ++++++++++++++++++++++++++++++- src/library/entry.cr | 14 ++------------ src/library/title.cr | 36 ++++++++++++++++++++++-------------- src/util/util.cr | 7 ++++++- 4 files changed, 60 insertions(+), 28 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index d0d3f016..686ae1fa 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -1,6 +1,7 @@ require "digest" require "./entry" +require "./title" require "./types" # Base class for an entry in the LRU cache. @@ -81,6 +82,31 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) end end +class SortedTitlesCacheEntry < CacheEntry(Array(String), Array(Title)) + def self.to_save_t(value : Array(Title)) + value.map &.id + end + + def self.to_return_t(value : Array(String)) + value.map { |title_id| Library.default.title_hash[title_id].not_nil! } + end + + def instance_size + instance_sizeof(SortedTitlesCacheEntry) + # sizeof itself + instance_sizeof(String) + @key.bytesize + # allocated memory for @key + @value.size * (instance_sizeof(String) + sizeof(String)) + + @value.sum(&.bytesize) # elements in Array(String) + end + + def self.gen_key(username : String, titles : Array(Title), opt : SortOptions?) + titles_sig = Digest::SHA1.hexdigest (titles.map &.id).to_s + user_context = opt && opt.method == SortMethod::Progress ? username : "" + sig = Digest::SHA1.hexdigest (titles_sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) + "#{sig}:sorted_titles" + end +end + class String def instance_size instance_sizeof(String) + bytesize @@ -101,14 +127,17 @@ struct Tuple(*T) end end -alias CacheableType = Array(Entry) | String | Tuple(String, Int32) +alias CacheableType = Array(Entry) | Array(Title) | String | Tuple(String, Int32) alias CacheEntryType = SortedEntriesCacheEntry | + SortedTitlesCacheEntry | CacheEntry(String, String) | CacheEntry(Tuple(String, Int32), Tuple(String, Int32)) def generate_cache_entry(key : String, value : CacheableType) if value.is_a? Array(Entry) SortedEntriesCacheEntry.new key, value + elsif value.is_a? Array(Title) + SortedTitlesCacheEntry.new key, value else CacheEntry(typeof(value), typeof(value)).new key, value end diff --git a/src/library/entry.cr b/src/library/entry.cr index ec11cb12..4c9c96ff 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -89,13 +89,7 @@ class Entry @sort_title = sort_title end - [false, true].each do |ascend| - [SortMethod::Auto, SortMethod::Title].each do |sort_method| - sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id, - username, @book.entries, SortOptions.new(sort_method, ascend) - LRUCache.invalidate sorted_entries_cache_key - end - end + @book.remove_sorted_caches [SortMethod::Auto, SortMethod::Title], username end def sort_title_db @@ -213,11 +207,7 @@ class Entry @book.parents.each do |parent| LRUCache.invalidate "#{parent.id}:#{username}:progress_sum" end - [false, true].each do |ascend| - sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id, - username, @book.entries, SortOptions.new(SortMethod::Progress, ascend) - LRUCache.invalidate sorted_entries_cache_key - end + @book.remove_sorted_caches [SortMethod::Progress], username TitleInfo.new @book.dir do |info| if info.progress[username]?.nil? diff --git a/src/library/title.cr b/src/library/title.cr index 9eacf4eb..f61f01d5 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -317,14 +317,7 @@ class Title @sort_title = sort_title end - [false, true].each do |ascend| - [SortMethod::Auto, SortMethod::Title].each do |sort_method| - sorted_entries_cache_key = - SortedEntriesCacheEntry.gen_key @id, username, @entries, - SortOptions.new(sort_method, ascend) - LRUCache.invalidate sorted_entries_cache_key - end - end + remove_sorted_caches [SortMethod::Auto, SortMethod::Title], username end def sort_title_db @@ -605,17 +598,32 @@ class Title zip + titles.flat_map &.deep_entries_with_date_added end + def remove_sorted_caches(sort_methods : Array(SortMethod), username : String) + [false, true].each do |ascend| + sort_methods.each do |sort_method| + sorted_entries_cache_key = + SortedEntriesCacheEntry.gen_key @id, username, @entries, + SortOptions.new(sort_method, ascend) + LRUCache.invalidate sorted_entries_cache_key + end + end + parents.each do |parent| + [false, true].each do |ascend| + sort_methods.each do |sort_method| + sorted_titles_cache_key = SortedTitlesCacheEntry.gen_key username, + parent.titles, SortOptions.new(sort_method, ascend) + LRUCache.invalidate sorted_titles_cache_key + end + end + end + end + def bulk_progress(action, ids : Array(String), username) LRUCache.invalidate "#{@id}:#{username}:progress_sum" parents.each do |parent| LRUCache.invalidate "#{parent.id}:#{username}:progress_sum" end - [false, true].each do |ascend| - sorted_entries_cache_key = - SortedEntriesCacheEntry.gen_key @id, username, @entries, - SortOptions.new(SortMethod::Progress, ascend) - LRUCache.invalidate sorted_entries_cache_key - end + remove_sorted_caches [SortMethod::Progress], username selected_entries = ids .map { |id| diff --git a/src/util/util.cr b/src/util/util.cr index 2814c6c4..74738b79 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -86,7 +86,11 @@ def env_is_true?(key : String) : Bool val.downcase.in? "1", "true" end -def sort_titles(titles : Array(Title), opt : SortOptions, username : String) +def sort_titles(titles : Array(Title), opt : SortOptions, username : String) : Array(Title) + cache_key = SortedTitlesCacheEntry.gen_key username, titles, opt + cached_titles = LRUCache.get cache_key + return cached_titles if cached_titles.is_a? Array(Title) + ary = titles case opt.method @@ -110,6 +114,7 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String) ary.reverse! unless opt.not_nil!.ascend + LRUCache.set generate_cache_entry cache_key, ary ary end From bd2ed1b338214911c806be30cf82b28cf8413617 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 26 Dec 2021 03:01:51 +0900 Subject: [PATCH 22/45] Implement to restore a display name with an empty input Add a placeholder for the default display name Remove some console.log() callings --- public/js/title.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/public/js/title.js b/public/js/title.js index 5a2da33e..d6cde9a6 100644 --- a/public/js/title.js +++ b/public/js/title.js @@ -90,8 +90,6 @@ const renameSubmit = (name, eid) => { const upload = $('.upload-field'); const titleId = upload.attr('data-title-id'); - console.log(name); - if (name.length === 0) { alert('danger', 'The display name should not be empty'); return; @@ -172,20 +170,19 @@ const edit = (eid) => { const displayNameField = $('#display-name-field'); displayNameField.attr('value', displayName); - console.log(displayNameField); + displayNameField.attr('placeholder', fileTitle); displayNameField.keyup(event => { if (event.keyCode === 13) { - renameSubmit(displayNameField.val(), eid); + renameSubmit(displayNameField.val() || fileTitle, eid); } }); displayNameField.siblings('a.uk-form-icon').click(() => { - renameSubmit(displayNameField.val(), eid); + renameSubmit(displayNameField.val() || fileTitle, eid); }); const sortTitleField = $('#sort-title-field'); sortTitleField.attr('value', sortTitle); sortTitleField.attr('placeholder', fileTitle); - console.log(sortTitle); sortTitleField.keyup(event => { if (event.keyCode === 13) { renameSortNameSubmit(sortTitleField.val(), eid); @@ -211,7 +208,6 @@ const setupUpload = (eid) => { queryObj['eid'] = eid; const query = $.param(queryObj); const url = `${base_url}api/admin/upload/cover?${query}`; - console.log(url); UIkit.upload('.upload-field', { url: url, name: 'file', From 0f94288bab36d88b905d7c7cd8d402f3e62cf2c4 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 26 Dec 2021 03:29:41 +0900 Subject: [PATCH 23/45] Avoid N+1 queries problem --- src/library/entry.cr | 5 +++-- src/library/title.cr | 14 +++++++++++++- src/storage.cr | 16 ++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 4c9c96ff..067acce7 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -72,7 +72,7 @@ class Entry def sort_title sort_title_cached = @sort_title return sort_title_cached if sort_title_cached - sort_title = Storage.default.get_entry_sort_title id + sort_title = @book.entry_sort_title_db id if sort_title @sort_title = sort_title return sort_title @@ -89,11 +89,12 @@ class Entry @sort_title = sort_title end + @book.entry_sort_title_cache = nil @book.remove_sorted_caches [SortMethod::Auto, SortMethod::Title], username end def sort_title_db - Storage.default.get_entry_sort_title id + @book.entry_sort_title_db @id end def display_name diff --git a/src/library/title.cr b/src/library/title.cr index f61f01d5..8465d567 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -8,11 +8,14 @@ class Title entries : Array(Entry), title : String, id : String, encoded_title : String, mtime : Time, signature : UInt64, entry_cover_url_cache : Hash(String, String)? - setter entry_cover_url_cache : Hash(String, String)? + setter entry_cover_url_cache : Hash(String, String)?, + entry_sort_title_cache : Hash(String, String | Nil)? @[YAML::Field(ignore: true)] @sort_title : String? @[YAML::Field(ignore: true)] + @entry_sort_title_cache : Hash(String, String | Nil)? + @[YAML::Field(ignore: true)] @entry_display_name_cache : Hash(String, String)? @[YAML::Field(ignore: true)] @entry_cover_url_cache : Hash(String, String)? @@ -324,6 +327,15 @@ class Title Storage.default.get_title_sort_title id end + def entry_sort_title_db(entry_id) + unless @entry_sort_title_cache + @entry_sort_title_cache = + Storage.default.get_entries_sort_title @entries.map &.id + end + + @entry_sort_title_cache.not_nil![entry_id]? + end + def tags Storage.default.get_title_tags @id end diff --git a/src/storage.cr b/src/storage.cr index edf71ff4..cd3e3c06 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -371,6 +371,22 @@ class Storage sort_title end + def get_entries_sort_title(ids : Array(String)) + results = Hash(String, String | Nil).new + MainFiber.run do + get_db do |db| + db.query "select id, sort_title from ids where id in (#{ids.join "," { |id| "'#{id}'" }})" do |rs| + rs.each do + id = rs.read String + sort_title = rs.read String | Nil + results[id] = sort_title + end + end + end + end + results + end + def set_entry_sort_title(entry_id : String, sort_title : String | Nil) sort_title = nil if sort_title == "" MainFiber.run do From b711072492bcc527e06d8679150a9c785813caa6 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 26 Dec 2021 04:10:03 +0900 Subject: [PATCH 24/45] Fix lint --- src/library/cache.cr | 3 ++- src/library/title.cr | 4 +++- src/routes/main.cr | 4 ++-- src/storage.cr | 17 ++++++++++++----- src/util/util.cr | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 686ae1fa..10e4f60f 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -127,7 +127,8 @@ struct Tuple(*T) end end -alias CacheableType = Array(Entry) | Array(Title) | String | Tuple(String, Int32) +alias CacheableType = Array(Entry) | Array(Title) | String | + Tuple(String, Int32) alias CacheEntryType = SortedEntriesCacheEntry | SortedTitlesCacheEntry | CacheEntry(String, String) | diff --git a/src/library/title.cr b/src/library/title.cr index 8465d567..d5d86927 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -522,7 +522,9 @@ class Title case opt.not_nil!.method when .title? - ary = @entries.sort { |a, b| compare_numerically a.sort_title, b.sort_title } + ary = @entries.sort do |a, b| + compare_numerically a.sort_title, b.sort_title + end when .time_modified? ary = @entries.sort { |a, b| (a.mtime <=> b.mtime).or \ compare_numerically a.sort_title, b.sort_title } diff --git a/src/routes/main.cr b/src/routes/main.cr index 87a7c3ba..ea2f0d8c 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -66,9 +66,9 @@ struct MainRouter percentage = title.load_percentage_for_all_entries username, sort_opt title_percentage = title.titles.map &.load_percentage username title_percentage_map = {} of String => Float64 - title_percentage.each_with_index do |percentage, i| + title_percentage.each_with_index do |tp, i| t = title.titles[i] - title_percentage_map[t.id] = percentage + title_percentage_map[t.id] = tp end layout "title" diff --git a/src/storage.cr b/src/storage.cr index cd3e3c06..4c8cfe7b 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -346,7 +346,9 @@ class Storage sort_title = nil MainFiber.run do get_db do |db| - sort_title = db.query_one? "Select sort_title from titles where id = (?)", title_id, as: String | Nil + sort_title = + db.query_one? "Select sort_title from titles where id = (?)", + title_id, as: String | Nil end end sort_title @@ -356,7 +358,8 @@ class Storage sort_title = nil if sort_title == "" MainFiber.run do get_db do |db| - db.exec "update titles set sort_title = (?) where id = (?)", sort_title, title_id + db.exec "update titles set sort_title = (?) where id = (?)", + sort_title, title_id end end end @@ -365,7 +368,9 @@ class Storage sort_title = nil MainFiber.run do get_db do |db| - sort_title = db.query_one? "Select sort_title from ids where id = (?)", entry_id, as: String | Nil + sort_title = + db.query_one? "Select sort_title from ids where id = (?)", + entry_id, as: String | Nil end end sort_title @@ -375,7 +380,8 @@ class Storage results = Hash(String, String | Nil).new MainFiber.run do get_db do |db| - db.query "select id, sort_title from ids where id in (#{ids.join "," { |id| "'#{id}'" }})" do |rs| + db.query "select id, sort_title from ids where id in " \ + "(#{ids.join "," { |id| "'#{id}'" }})" do |rs| rs.each do id = rs.read String sort_title = rs.read String | Nil @@ -391,7 +397,8 @@ class Storage sort_title = nil if sort_title == "" MainFiber.run do get_db do |db| - db.exec "update ids set sort_title = (?) where id = (?)", sort_title, entry_id + db.exec "update ids set sort_title = (?) where id = (?)", + sort_title, entry_id end end end diff --git a/src/util/util.cr b/src/util/util.cr index 74738b79..82edb6a6 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -86,7 +86,7 @@ def env_is_true?(key : String) : Bool val.downcase.in? "1", "true" end -def sort_titles(titles : Array(Title), opt : SortOptions, username : String) : Array(Title) +def sort_titles(titles : Array(Title), opt : SortOptions, username : String) cache_key = SortedTitlesCacheEntry.gen_key username, titles, opt cached_titles = LRUCache.get cache_key return cached_titles if cached_titles.is_a? Array(Title) From ecaec307d6549c03e84ea333bd973087d31d5ace Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 26 Dec 2021 04:27:27 +0900 Subject: [PATCH 25/45] Fix title sort bug, invalidate titles of the Library Refactor remove cache --- src/library/entry.cr | 3 ++- src/library/title.cr | 24 +++++++++++++++--------- src/util/util.cr | 24 ++++++++++++++++++------ 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 067acce7..8b80d46f 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -90,7 +90,8 @@ class Entry end @book.entry_sort_title_cache = nil - @book.remove_sorted_caches [SortMethod::Auto, SortMethod::Title], username + @book.remove_sorted_entries_cache [SortMethod::Auto, SortMethod::Title], + username end def sort_title_db diff --git a/src/library/title.cr b/src/library/title.cr index d5d86927..6fa78a55 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -320,7 +320,13 @@ class Title @sort_title = sort_title end - remove_sorted_caches [SortMethod::Auto, SortMethod::Title], username + if parents.size > 0 + target = parents[-1].titles + else + target = Library.default.titles + end + remove_sorted_titles_cache target, + [SortMethod::Auto, SortMethod::Title], username end def sort_title_db @@ -612,7 +618,8 @@ class Title zip + titles.flat_map &.deep_entries_with_date_added end - def remove_sorted_caches(sort_methods : Array(SortMethod), username : String) + def remove_sorted_entries_cache(sort_methods : Array(SortMethod), + username : String) [false, true].each do |ascend| sort_methods.each do |sort_method| sorted_entries_cache_key = @@ -621,15 +628,14 @@ class Title LRUCache.invalidate sorted_entries_cache_key end end + end + + def remove_sorted_caches(sort_methods : Array(SortMethod), username : String) + remove_sorted_entries_cache sort_methods, username parents.each do |parent| - [false, true].each do |ascend| - sort_methods.each do |sort_method| - sorted_titles_cache_key = SortedTitlesCacheEntry.gen_key username, - parent.titles, SortOptions.new(sort_method, ascend) - LRUCache.invalidate sorted_titles_cache_key - end - end + remove_sorted_titles_cache parent.titles, sort_methods, username end + remove_sorted_titles_cache Library.default.titles, sort_methods, username end def bulk_progress(action, ids : Array(String), username) diff --git a/src/util/util.cr b/src/util/util.cr index 82edb6a6..11d1a137 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -91,25 +91,25 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String) cached_titles = LRUCache.get cache_key return cached_titles if cached_titles.is_a? Array(Title) - ary = titles - case opt.method when .time_modified? - ary.sort { |a, b| (a.mtime <=> b.mtime).or \ + ary = titles.sort { |a, b| (a.mtime <=> b.mtime).or \ compare_numerically a.sort_title, b.sort_title } when .progress? - ary.sort do |a, b| + ary = titles.sort do |a, b| (a.load_percentage(username) <=> b.load_percentage(username)).or \ compare_numerically a.sort_title, b.sort_title end when .title? - ary.sort { |a, b| compare_numerically a.sort_title, b.sort_title } + ary = titles.sort do |a, b| + compare_numerically a.sort_title, b.sort_title + end else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ "Auto instead" end - ary.sort { |a, b| compare_numerically a.sort_title, b.sort_title } + ary = titles.sort { |a, b| compare_numerically a.sort_title, b.sort_title } end ary.reverse! unless opt.not_nil!.ascend @@ -118,6 +118,18 @@ def sort_titles(titles : Array(Title), opt : SortOptions, username : String) ary end +def remove_sorted_titles_cache(titles : Array(Title), + sort_methods : Array(SortMethod), + username : String) + [false, true].each do |ascend| + sort_methods.each do |sort_method| + sorted_titles_cache_key = SortedTitlesCacheEntry.gen_key username, + titles, SortOptions.new(sort_method, ascend) + LRUCache.invalidate sorted_titles_cache_key + end + end +end + class String # Returns the similarity (in [0, 1]) of two paths. # For the two paths, separate them into arrays of components, count the From e7538bb7f20a791840d45471a2dd97fea8658ae7 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 26 Dec 2021 05:56:45 +0900 Subject: [PATCH 26/45] Use val(), Remove callbacks after modal hidden --- public/js/title.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/public/js/title.js b/public/js/title.js index d6cde9a6..5d0e49f2 100644 --- a/public/js/title.js +++ b/public/js/title.js @@ -60,6 +60,11 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi UIkit.modal($('#modal')).show(); } +UIkit.util.on(document, 'hidden', '#modal', () => { + $('#read-btn').off('click'); + $('#unread-btn').off('click'); +}); + const updateProgress = (tid, eid, page) => { let url = `${base_url}api/progress/${tid}/${page}` const query = $.param({ @@ -181,7 +186,7 @@ const edit = (eid) => { }); const sortTitleField = $('#sort-title-field'); - sortTitleField.attr('value', sortTitle); + sortTitleField.val(sortTitle); sortTitleField.attr('placeholder', fileTitle); sortTitleField.keyup(event => { if (event.keyCode === 13) { @@ -197,6 +202,16 @@ const edit = (eid) => { UIkit.modal($('#edit-modal')).show(); }; +UIkit.util.on(document, 'hidden', '#edit-modal', () => { + const displayNameField = $('#display-name-field'); + displayNameField.off('keyup'); + displayNameField.off('click'); + + const sortTitleField = $('#sort-title-field'); + sortTitleField.off('keyup'); + sortTitleField.off('click'); +}); + const setupUpload = (eid) => { const upload = $('.upload-field'); const bar = $('#upload-progress').get(0); From ab29a9eb80daf729960f0fcc3dbc96b34f4fb8ef Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 31 Dec 2021 14:33:49 +0000 Subject: [PATCH 27/45] Fix down SQL for removing columns --- migration/sort_title.12.cr | 83 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/migration/sort_title.12.cr b/migration/sort_title.12.cr index 8da828fb..a62f2447 100644 --- a/migration/sort_title.12.cr +++ b/migration/sort_title.12.cr @@ -9,9 +9,86 @@ class SortTitle < MG::Base def down : String <<-SQL - -- drop sort_title column to ids and titles - ALTER TABLE ids DROP COLUMN sort_title; - ALTER TABLE titles DROP COLUMN sort_title; + -- remove sort_title column from ids + ALTER TABLE ids RENAME TO tmp; + + CREATE TABLE ids ( + path TEXT NOT NULL, + id TEXT NOT NULL, + signature TEXT, + unavailable INTEGER NOT NULL DEFAULT 0 + ); + + INSERT INTO ids + SELECT path, id, signature, unavailable + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX path_idx ON ids (path); + CREATE UNIQUE INDEX id_idx ON ids (id); + + -- recreate the foreign key constraint on thumbnails + ALTER TABLE thumbnails RENAME TO tmp; + + CREATE TABLE thumbnails ( + id TEXT NOT NULL, + data BLOB NOT NULL, + filename TEXT NOT NULL, + mime TEXT NOT NULL, + size INTEGER NOT NULL, + FOREIGN KEY (id) REFERENCES ids (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO thumbnails + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE UNIQUE INDEX tn_index ON thumbnails (id); + + -- remove sort_title column from titles + ALTER TABLE titles RENAME TO tmp; + + CREATE TABLE titles ( + id TEXT NOT NULL, + path TEXT NOT NULL, + signature TEXT, + unavailable INTEGER NOT NULL DEFAULT 0 + ); + + INSERT INTO titles + SELECT path, id, signature, unavailable + FROM tmp; + + DROP TABLE tmp; + + -- recreate the indices + CREATE UNIQUE INDEX titles_id_idx on titles (id); + CREATE UNIQUE INDEX titles_path_idx on titles (path); + + -- recreate the foreign key constraint on tags + ALTER TABLE tags RENAME TO tmp; + + CREATE TABLE tags ( + id TEXT NOT NULL, + tag TEXT NOT NULL, + UNIQUE (id, tag), + FOREIGN KEY (id) REFERENCES titles (id) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + + INSERT INTO tags + SELECT * FROM tmp; + + DROP TABLE tmp; + + CREATE INDEX tags_id_idx ON tags (id); + CREATE INDEX tags_tag_idx ON tags (tag); SQL end end From 4b302af2a199b61d721093fba7362b86696455fd Mon Sep 17 00:00:00 2001 From: Robbo Date: Tue, 4 Jan 2022 00:20:52 +1100 Subject: [PATCH 28/45] Add Right to Left option to Paged viewing mode --- public/js/reader.js | 14 +++++++++++++- src/views/reader.html.ecr | 10 ++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/public/js/reader.js b/public/js/reader.js index 9b172766..0b927cb6 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -13,6 +13,7 @@ const readerComponent = () => { selectedIndex: 0, // 0: not selected; 1: the first page margin: 30, preloadLookahead: 3, + enableRightToLeft: false, /** * Initialize the component by fetching the page dimensions @@ -64,6 +65,13 @@ const readerComponent = () => { const savedFlipAnimation = localStorage.getItem('enableFlipAnimation'); this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true'; + + const savedRightToLeft = localStorage.getItem('enableRightToLeft'); + if (savedRightToLeft === null) { + this.enableRightToLeft = false; + } else { + this.enableRightToLeft = savedRightToLeft; + } }) .catch(e => { const errMsg = `Failed to get the page dimensions. ${e}`; @@ -136,7 +144,7 @@ const readerComponent = () => { this.toPage(newIdx); if (this.enableFlipAnimation) { - if (isNext) + if (isNext ^ this.enableRightToLeft) this.flipAnimation = 'right'; else this.flipAnimation = 'left'; @@ -320,5 +328,9 @@ const readerComponent = () => { enableFlipAnimationChanged() { localStorage.setItem('enableFlipAnimation', this.enableFlipAnimation); }, + + enableRightToLeftChanged() { + localStorage.setItem('enableRightToLeft', this.enableRightToLeft); + }, }; } diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index 0e46ed20..395de413 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -55,8 +55,8 @@ object-fit: contain; `" /> -
-
+
+
@@ -114,6 +114,12 @@
+
+ +
+ +
+
From 27cc669012c93b3109de8555795c96a4146bb58f Mon Sep 17 00:00:00 2001 From: Robbo Date: Tue, 4 Jan 2022 20:43:55 +1100 Subject: [PATCH 29/45] Fix Right to Left for keyboard input --- public/js/reader.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/js/reader.js b/public/js/reader.js index 0b927cb6..b20f1cd7 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -122,9 +122,9 @@ const readerComponent = () => { if (this.mode === 'continuous') return; if (event.key === 'ArrowLeft' || event.key === 'k') - this.flipPage(false); + this.flipPage(false ^ this.enableRightToLeft); if (event.key === 'ArrowRight' || event.key === 'j') - this.flipPage(true); + this.flipPage(true ^ this.enableRightToLeft); }, /** * Flips to the next or the previous page From 86beed0c5f396514e5f0b7102e836725d7240228 Mon Sep 17 00:00:00 2001 From: Robbo Date: Wed, 5 Jan 2022 18:45:15 +1100 Subject: [PATCH 30/45] Cast RightToLeft value to boolean when retrieving from local storage. --- public/js/reader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/js/reader.js b/public/js/reader.js index b20f1cd7..2cb3a666 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -70,7 +70,7 @@ const readerComponent = () => { if (savedRightToLeft === null) { this.enableRightToLeft = false; } else { - this.enableRightToLeft = savedRightToLeft; + this.enableRightToLeft = (savedRightToLeft === 'true'); } }) .catch(e => { From b83ccf1ccce354352d847531cfb005a10443b8ab Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 15 Jan 2022 12:08:23 +0000 Subject: [PATCH 31/45] Fix down SQL --- migration/sort_title.12.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration/sort_title.12.cr b/migration/sort_title.12.cr index a62f2447..9853182b 100644 --- a/migration/sort_title.12.cr +++ b/migration/sort_title.12.cr @@ -61,7 +61,7 @@ class SortTitle < MG::Base ); INSERT INTO titles - SELECT path, id, signature, unavailable + SELECT id, path, signature, unavailable FROM tmp; DROP TABLE tmp; From 1b32dc3de9d024295023c13052fe7cba75882434 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 15 Jan 2022 12:09:03 +0000 Subject: [PATCH 32/45] Add sort title to API response --- src/library/entry.cr | 1 + src/library/title.cr | 1 + 2 files changed, 2 insertions(+) diff --git a/src/library/entry.cr b/src/library/entry.cr index 8b80d46f..ceaa5311 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -59,6 +59,7 @@ class Entry json.field {{str}}, @{{str.id}} {% end %} json.field "title_id", @book.id + json.field "sort_title", sort_title json.field "pages" { json.number @pages } unless slim json.field "display_name", @book.display_name @title diff --git a/src/library/title.cr b/src/library/title.cr index 6fa78a55..539f114f 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -209,6 +209,7 @@ class Title json.field {{str}}, @{{str.id}} {% end %} json.field "signature" { json.number @signature } + json.field "sort_title", sort_title unless slim json.field "display_name", display_name json.field "cover_url", cover_url From b2329a79b411af393af90853dae3c6ad9005e361 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 18 Jan 2022 15:02:16 +0000 Subject: [PATCH 33/45] Gracefully handle nullish fields --- public/js/plugin-download.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/js/plugin-download.js b/public/js/plugin-download.js index 2346fe27..11c047ce 100644 --- a/public/js/plugin-download.js +++ b/public/js/plugin-download.js @@ -70,7 +70,7 @@ const buildTable = (chapters) => { const rows = chapters.map(ch => { const tds = Object.values(ch).map(v => { const maxLength = 40; - const shouldShrink = v.length > maxLength; + const shouldShrink = v && v.length > maxLength; const content = shouldShrink ? `${v.substring(0, maxLength)}...
${v}
` : v; return `${content}` }).join(''); From 750fbbb8fec3a0c2ca70e8534218fa68771219cb Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 24 Jan 2022 13:25:55 +0000 Subject: [PATCH 34/45] Delete cache when dir mismatch (fixes #265) --- src/library/library.cr | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/library/library.cr b/src/library/library.cr index a2dc28d9..bd801acb 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -24,7 +24,21 @@ class Library begin Compress::Gzip::Reader.open path do |content| - @@default = Library.from_yaml content + loaded = Library.from_yaml content + if loaded.dir != Config.current.library_path + # We will have to do a full restart in this case. Otherwise having + # two instances of the library will cause some weirdness. + Logger.fatal "Cached library dir #{loaded.dir} does not match " \ + "current library dir #{Config.current.library_path}. " \ + "Deleting cache" + File.delete path + Logger.fatal "Invalid library cache deleted. Mango needs to " \ + "perform a full reset to recover from this. " \ + "Pleae restart Mango." + Logger.fatal "Exiting" + exit 1 + end + @@default = loaded end Library.default.register_jobs rescue e From 77df4183905a4ae8bc9e554baaff8c874ed90d4e Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 24 Jan 2022 14:18:52 +0000 Subject: [PATCH 35/45] Compare with DB when loading library cache (fixes #256) --- src/library/library.cr | 18 ++++++++++-------- src/storage.cr | 14 ++++++++++++++ src/util/util.cr | 9 +++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index bd801acb..7c9c020f 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -25,20 +25,22 @@ class Library begin Compress::Gzip::Reader.open path do |content| loaded = Library.from_yaml content + # We will have to do a full restart in these cases. Otherwise having + # two instances of the library will cause some weirdness. if loaded.dir != Config.current.library_path - # We will have to do a full restart in this case. Otherwise having - # two instances of the library will cause some weirdness. Logger.fatal "Cached library dir #{loaded.dir} does not match " \ "current library dir #{Config.current.library_path}. " \ "Deleting cache" - File.delete path - Logger.fatal "Invalid library cache deleted. Mango needs to " \ - "perform a full reset to recover from this. " \ - "Pleae restart Mango." - Logger.fatal "Exiting" - exit 1 + delete_cache_and_exit path + end + if loaded.title_ids.size > 0 && + Storage.default.count_titles == 0 + Logger.fatal "The library cache is inconsistent with the DB. " \ + "Deleting cache" + delete_cache_and_exit path end @@default = loaded + Logger.debug "Library cache loaded" end Library.default.register_jobs rescue e diff --git a/src/storage.cr b/src/storage.cr index 4c8cfe7b..5622241c 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -619,6 +619,20 @@ class Storage {token, expires} end + def count_titles : Int32 + count = 0 + MainFiber.run do + get_db do |db| + db.query "select count(*) from titles" do |rs| + rs.each do + count = rs.read Int32 + end + end + end + end + count + end + def close MainFiber.run do unless @db.nil? diff --git a/src/util/util.cr b/src/util/util.cr index 11d1a137..e7b1b1aa 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -163,3 +163,12 @@ def sanitize_filename(str : String) : String .gsub(/[\177\000-\031\\:\*\?\"<>\|]/, "") sanitized.size > 0 ? sanitized : random_str end + +def delete_cache_and_exit(path : String) + File.delete path + Logger.fatal "Invalid library cache deleted. Mango needs to " \ + "perform a full reset to recover from this. " \ + "Pleae restart Mango. This is NOT a bug." + Logger.fatal "Exiting" + exit 1 +end From fea6c04c4f2490286bbcde8bef7931bb4229cfe6 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 23 Jan 2022 08:46:19 +0000 Subject: [PATCH 36/45] Fix actions on download manager (fixes #266) --- public/js/download-manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/js/download-manager.js b/public/js/download-manager.js index 0393dd37..1183ce53 100644 --- a/public/js/download-manager.js +++ b/public/js/download-manager.js @@ -55,7 +55,7 @@ const component = () => { jobAction(action, event) { let url = `${base_url}api/admin/mangadex/queue/${action}`; if (event) { - const id = event.currentTarget.closest('tr').id.split('-')[1]; + const id = event.currentTarget.closest('tr').id.split('-').slice(1).join('-'); url = `${url}?${$.param({ id: id })}`; From 9b111b0ee880e9c2f2f0bb32b6e69cc8d5ad6809 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Wed, 26 Jan 2022 09:10:47 +0000 Subject: [PATCH 37/45] Ignore thumbnail progress in cache (fixes #270) --- src/library/library.cr | 53 +++++++++++++++++++++++++++++------------- src/routes/api.cr | 2 +- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 7c9c020f..210912f4 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -1,9 +1,38 @@ class Library + struct ThumbnailContext + property current : Int32, total : Int32 + + def initialize + @current = 0 + @total = 0 + end + + def progress + if total == 0 + 0 + else + current / total + end + end + + def reset + @current = 0 + @total = 0 + end + + def increment + @current += 1 + end + end + include YAML::Serializable getter dir : String, title_ids : Array(String), title_hash : Hash(String, Title) + @[YAML::Field(ignore: true)] + getter thumbnail_ctx = ThumbnailContext.new + use_default def save_instance @@ -55,9 +84,6 @@ class Library @title_ids = [] of String @title_hash = {} of String => Title - @entries_count = 0 - @thumbnails_count = 0 - register_jobs end @@ -278,34 +304,29 @@ class Library .shuffle! end - def thumbnail_generation_progress - return 0 if @entries_count == 0 - @thumbnails_count / @entries_count - end - def generate_thumbnails - if @thumbnails_count > 0 + if thumbnail_ctx.current > 0 Logger.debug "Thumbnail generation in progress" return end Logger.info "Starting thumbnail generation" entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg - @entries_count = entries.size - @thumbnails_count = 0 + thumbnail_ctx.total = entries.size + thumbnail_ctx.current = 0 # Report generation progress regularly spawn do loop do - unless @thumbnails_count == 0 + unless thumbnail_ctx.current == 0 Logger.debug "Thumbnail generation progress: " \ - "#{(thumbnail_generation_progress * 100).round 1}%" + "#{(thumbnail_ctx.progress * 100).round 1}%" end # Generation is completed. We reset the count to 0 to allow subsequent # calls to the function, and break from the loop to stop the progress # report fiber - if thumbnail_generation_progress.to_i == 1 - @thumbnails_count = 0 + if thumbnail_ctx.progress.to_i == 1 + thumbnail_ctx.reset break end sleep 10.seconds @@ -319,7 +340,7 @@ class Library # and CPU sleep 1.seconds end - @thumbnails_count += 1 + thumbnail_ctx.increment end Logger.info "Thumbnail generation finished" end diff --git a/src/routes/api.cr b/src/routes/api.cr index 28ec314b..413c318b 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -240,7 +240,7 @@ struct APIRouter } get "/api/admin/thumbnail_progress" do |env| send_json env, { - "progress" => Library.default.thumbnail_generation_progress, + "progress" => Library.default.thumbnail_ctx.progress, }.to_json end From 167e207fad05a7388d7d2d3f1b9431278b417da2 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Wed, 26 Jan 2022 12:12:01 +0000 Subject: [PATCH 38/45] Bump version to 0.25.0 --- README.md | 2 +- shard.yml | 2 +- src/mango.cr | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b4d3d3a1..5a12dc69 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.24.0 + Mango - Manga Server and Web Reader. Version 0.25.0 Usage: diff --git a/shard.yml b/shard.yml index 0054a230..44a09242 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.24.0 +version: 0.25.0 authors: - Alex Ling diff --git a/src/mango.cr b/src/mango.cr index 8716d044..3cdafc05 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -7,7 +7,7 @@ require "option_parser" require "clim" require "tallboy" -MANGO_VERSION = "0.24.0" +MANGO_VERSION = "0.25.0" # From http://www.network-science.de/ascii/ BANNER = %{ From a29d6754e8a6895e4238c2cd90c64bb9a47a94c3 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 6 Feb 2022 06:17:42 +0000 Subject: [PATCH 39/45] Expand paths in config (closes #277) --- src/config.cr | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/config.cr b/src/config.cr index aa818c32..f44e3311 100644 --- a/src/config.cr +++ b/src/config.cr @@ -9,18 +9,14 @@ class Config property port : Int32 = 9000 property base_url : String = "/" property session_secret : String = "mango-session-secret" - property library_path : String = File.expand_path "~/mango/library", - home: true - property library_cache_path = File.expand_path "~/mango/library.yml.gz", - home: true - property db_path : String = File.expand_path "~/mango/mango.db", home: true + property library_path : String = "~/mango/library" + property library_cache_path = "~/mango/library.yml.gz" + property db_path : String = "~/mango/mango.db" property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 property log_level : String = "info" - property upload_path : String = File.expand_path "~/mango/uploads", - home: true - property plugin_path : String = File.expand_path "~/mango/plugins", - home: true + property upload_path : String = "~/mango/uploads" + property plugin_path : String = "~/mango/plugins" property download_timeout_seconds : Int32 = 30 property cache_enabled = false property cache_size_mbs = 50 @@ -59,6 +55,7 @@ class Config config = self.from_yaml File.read cfg_path config.path = path config.fill_defaults + config.expand_paths config.preprocess return config end @@ -67,6 +64,7 @@ class Config default = self.allocate default.path = path default.fill_defaults + default.expand_paths cfg_dir = File.dirname cfg_path unless Dir.exists? cfg_dir Dir.mkdir_p cfg_dir @@ -86,6 +84,12 @@ class Config {% end %} end + def expand_paths + {% for p in %w(library library_cache db upload plugin) %} + @{{p.id}}_path = File.expand_path @{{p.id}}_path, home: true + {% end %} + end + def preprocess unless base_url.starts_with? "/" raise "base url (#{base_url}) should start with `/`" From 59528de44da3847e4c08f77d01b473fe5a924227 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 6 Feb 2022 06:18:09 +0000 Subject: [PATCH 40/45] Remove `mangadex` entry from config --- src/config.cr | 47 ++--------------------------- src/queue.cr | 2 +- src/routes/admin.cr | 1 - src/views/download-manager.html.ecr | 6 ---- 4 files changed, 3 insertions(+), 53 deletions(-) diff --git a/src/config.cr b/src/config.cr index f44e3311..a5ad5d4b 100644 --- a/src/config.cr +++ b/src/config.cr @@ -12,6 +12,7 @@ class Config property library_path : String = "~/mango/library" property library_cache_path = "~/mango/library.yml.gz" property db_path : String = "~/mango/mango.db" + property queue_db_path : String = "~/mango/queue.db" property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 property log_level : String = "info" @@ -24,19 +25,6 @@ class Config property disable_login = false property default_username = "" property auth_proxy_header_name = "" - property mangadex = Hash(String, String | Int32).new - - @[YAML::Field(ignore: true)] - @mangadex_defaults = { - "base_url" => "https://mangadex.org", - "api_url" => "https://api.mangadex.org/v2", - "download_wait_seconds" => 5, - "download_retries" => 4, - "download_queue_db_path" => File.expand_path("~/mango/queue.db", - home: true), - "chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}", - "manga_rename_rule" => "{title}", - } @@singlet : Config? @@ -54,7 +42,6 @@ class Config if File.exists? cfg_path config = self.from_yaml File.read cfg_path config.path = path - config.fill_defaults config.expand_paths config.preprocess return config @@ -63,7 +50,6 @@ class Config "Dumping the default config there." default = self.allocate default.path = path - default.fill_defaults default.expand_paths cfg_dir = File.dirname cfg_path unless Dir.exists? cfg_dir @@ -74,18 +60,8 @@ class Config default end - def fill_defaults - {% for hash_name in ["mangadex"] %} - @{{hash_name.id}}_defaults.map do |k, v| - if @{{hash_name.id}}[k]?.nil? - @{{hash_name.id}}[k] = v - end - end - {% end %} - end - def expand_paths - {% for p in %w(library library_cache db upload plugin) %} + {% for p in %w(library library_cache db queue_db upload plugin) %} @{{p.id}}_path = File.expand_path @{{p.id}}_path, home: true {% end %} end @@ -101,24 +77,5 @@ class Config raise "Login is disabled, but default username is not set. " \ "Please set a default username" end - - # `Logger.default` is not available yet - Log.setup :debug - unless mangadex["api_url"] =~ /\/v2/ - Log.warn { "It looks like you are using the deprecated MangaDex API " \ - "v1 in your config file. Please update it to " \ - "https://api.mangadex.org/v2 to suppress this warning." } - mangadex["api_url"] = "https://api.mangadex.org/v2" - end - if mangadex["api_url"] =~ /\/api\/v2/ - Log.warn { "It looks like you are using the outdated MangaDex API " \ - "url (mangadex.org/api/v2) in your config file. Please " \ - "update it to https://api.mangadex.org/v2 to suppress this " \ - "warning." } - mangadex["api_url"] = "https://api.mangadex.org/v2" - end - - mangadex["api_url"] = mangadex["api_url"].to_s.rstrip "/" - mangadex["base_url"] = mangadex["base_url"].to_s.rstrip "/" end end diff --git a/src/queue.cr b/src/queue.cr index 381441b6..01cef38c 100644 --- a/src/queue.cr +++ b/src/queue.cr @@ -112,7 +112,7 @@ class Queue use_default def initialize(db_path : String? = nil) - @path = db_path || Config.current.mangadex["download_queue_db_path"].to_s + @path = db_path || Config.current.queue_db_path.to_s dir = File.dirname @path unless Dir.exists? dir Logger.info "The queue DB directory #{dir} does not exist. " \ diff --git a/src/routes/admin.cr b/src/routes/admin.cr index fd63ec82..a63bc0eb 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -66,7 +66,6 @@ struct AdminRouter end get "/admin/downloads" do |env| - mangadex_base_url = Config.current.mangadex["base_url"] layout "download-manager" end diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index c264177b..a8394ff0 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -24,16 +24,10 @@ - - From 84168b4f538a22523c6be17cffd5d9bbf2503241 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 6 Feb 2022 06:28:10 +0000 Subject: [PATCH 41/45] Update config example in README --- README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index 5a12dc69..5765c5e3 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ base_url: / session_secret: mango-session-secret library_path: ~/mango/library db_path: ~/mango/mango.db +queue_db_path: ~/mango/queue.db scan_interval_minutes: 5 thumbnail_generation_interval_hours: 24 log_level: info @@ -93,14 +94,6 @@ cache_log_enabled: true disable_login: false default_username: "" auth_proxy_header_name: "" -mangadex: - base_url: https://mangadex.org - api_url: https://api.mangadex.org/v2 - download_wait_seconds: 5 - download_retries: 4 - download_queue_db_path: ~/mango/queue.db - chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}' - manga_rename_rule: '{title}' ``` - `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks From b3ea3c6154f3b2d371f5164d250e2b3096e915c7 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 6 Feb 2022 06:28:39 +0000 Subject: [PATCH 42/45] Remove unnecessary type restrictions in config --- src/config.cr | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/config.cr b/src/config.cr index a5ad5d4b..c0f7d6a5 100644 --- a/src/config.cr +++ b/src/config.cr @@ -4,20 +4,20 @@ class Config include YAML::Serializable @[YAML::Field(ignore: true)] - property path : String = "" - property host : String = "0.0.0.0" + property path = "" + property host = "0.0.0.0" property port : Int32 = 9000 - property base_url : String = "/" - property session_secret : String = "mango-session-secret" - property library_path : String = "~/mango/library" + property base_url = "/" + property session_secret = "mango-session-secret" + property library_path = "~/mango/library" property library_cache_path = "~/mango/library.yml.gz" - property db_path : String = "~/mango/mango.db" - property queue_db_path : String = "~/mango/queue.db" + property db_path = "~/mango/mango.db" + property queue_db_path = "~/mango/queue.db" property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 - property log_level : String = "info" - property upload_path : String = "~/mango/uploads" - property plugin_path : String = "~/mango/plugins" + property log_level = "info" + property upload_path = "~/mango/uploads" + property plugin_path = "~/mango/plugins" property download_timeout_seconds : Int32 = 30 property cache_enabled = false property cache_size_mbs = 50 From 857c11be850580a39e19f29b90a6f0d198f37321 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 6 Feb 2022 06:39:46 +0000 Subject: [PATCH 43/45] Enable metadata cache by default --- src/config.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.cr b/src/config.cr index c0f7d6a5..b5b77dbf 100644 --- a/src/config.cr +++ b/src/config.cr @@ -19,7 +19,7 @@ class Config property upload_path = "~/mango/uploads" property plugin_path = "~/mango/plugins" property download_timeout_seconds : Int32 = 30 - property cache_enabled = false + property cache_enabled = true property cache_size_mbs = 50 property cache_log_enabled = true property disable_login = false From ab3386546dd46edcb3d44ba6863d7b3f63b6a797 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 6 Feb 2022 06:39:59 +0000 Subject: [PATCH 44/45] Remove db_optimization from README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5765c5e3..53cea050 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ default_username: "" auth_proxy_header_name: "" ``` -- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks +- `scan_interval_minutes`, `thumbnail_generation_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging - You can disable authentication by setting `disable_login` to true. Note that `default_username` must be set to an existing username for this to work. - By setting `cache_enabled` to `true`, you can enable an experimental feature where Mango caches library metadata to improve page load time. You can further fine-tune the feature with `cache_size_mbs` and `cache_log_enabled`. From eca47e3d3216ac1f3c863d39d4386469c93ae960 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 11 Feb 2022 14:28:05 +0000 Subject: [PATCH 45/45] Update README config example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 53cea050..e89a9a7d 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ upload_path: ~/mango/uploads plugin_path: ~/mango/plugins download_timeout_seconds: 30 library_cache_path: ~/mango/library.yml.gz -cache_enabled: false +cache_enabled: true cache_size_mbs: 50 cache_log_enabled: true disable_login: false