diff --git a/assets/css/darktheme.css b/assets/css/darktheme.css index 1b70956bb..ce478e386 100644 --- a/assets/css/darktheme.css +++ b/assets/css/darktheme.css @@ -21,10 +21,8 @@ body { color: #f0f0f0; } -.pure-form > fieldset > input, -.pure-control-group > input, -.pure-form > fieldset > select, -.pure-control-group > select { +input, +select { color: rgba(35, 35, 35, 1); } diff --git a/assets/js/embed.js b/assets/js/embed.js index d9af1f5b6..074a9d8da 100644 --- a/assets/js/embed.js +++ b/assets/js/embed.js @@ -12,7 +12,8 @@ function get_playlist(plid, retries) { '&format=html&hl=' + video_data.preferences.locale; } else { var plid_url = '/api/v1/playlists/' + plid + - '?continuation=' + video_data.id + + '?index=' + video_data.index + + '&continuation' + video_data.id + '&format=html&hl=' + video_data.preferences.locale; } @@ -45,6 +46,9 @@ function get_playlist(plid, retries) { } url.searchParams.set('list', plid); + if (!plid.startsWith('RD')) { + url.searchParams.set('index', xhr.response.index); + } location.assign(url.pathname + url.search); }); } diff --git a/assets/js/watch.js b/assets/js/watch.js index 0f3e81232..80cb1769a 100644 --- a/assets/js/watch.js +++ b/assets/js/watch.js @@ -133,7 +133,8 @@ function get_playlist(plid, retries) { '&format=html&hl=' + video_data.preferences.locale; } else { var plid_url = '/api/v1/playlists/' + plid + - '?continuation=' + video_data.id + + '?index=' + video_data.index + + '&continuation=' + video_data.id + '&format=html&hl=' + video_data.preferences.locale; } @@ -168,6 +169,9 @@ function get_playlist(plid, retries) { } url.searchParams.set('list', plid); + if (!plid.startsWith('RD')) { + url.searchParams.set('index', xhr.response.index); + } location.assign(url.pathname + url.search); }); } diff --git a/config/sql/playlist_videos.sql b/config/sql/playlist_videos.sql index d439c24af..b2b8d5c44 100644 --- a/config/sql/playlist_videos.sql +++ b/config/sql/playlist_videos.sql @@ -1,2 +1,19 @@ --- TODO: Playlist stub, add playlist thumbnail(?) -create table playlist_videos (title text, id text, author text, ucid text, length_seconds integer, published timestamptz, plid text references playlists(id), index int8, live_now boolean, primary key (index,plid)); +-- Table: public.playlist_videos + +-- DROP TABLE public.playlist_videos; + +CREATE TABLE playlist_videos +( + title text, + id text, + author text, + ucid text, + length_seconds integer, + published timestamptz, + plid text references playlists(id), + index int8, + live_now boolean, + PRIMARY KEY (index,plid) +); + +GRANT ALL ON TABLE public.playlist_videos TO kemal; diff --git a/config/sql/playlists.sql b/config/sql/playlists.sql index ce2751f3e..46ff30ecf 100644 --- a/config/sql/playlists.sql +++ b/config/sql/playlists.sql @@ -1,3 +1,18 @@ --- TODO: Playlist stub, check for missing enum when check_tables: true -create type privacy as enum ('Public', 'Unlisted', 'Private'); -create table playlists (title text, id text primary key, author text, description text, video_count integer, created timestamptz, updated timestamptz, privacy privacy, index int8[]); +-- Table: public.playlists + +-- DROP TABLE public.playlists; + +CREATE TABLE public.playlists +( + title text, + id text primary key, + author text, + description text, + video_count integer, + created timestamptz, + updated timestamptz, + privacy privacy, + index int8[] +); + +GRANT ALL ON public.playlists TO kemal; diff --git a/config/sql/privacy.sql b/config/sql/privacy.sql new file mode 100644 index 000000000..4356813ed --- /dev/null +++ b/config/sql/privacy.sql @@ -0,0 +1,10 @@ +-- Type: public.privacy + +-- DROP TYPE public.privacy; + +CREATE TYPE public.privacy AS ENUM +( + 'Public', + 'Unlisted', + 'Private' +); diff --git a/src/invidious.cr b/src/invidious.cr index 76cf79921..5e3e2acb6 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -125,17 +125,19 @@ Kemal::CLI.new ARGV # Check table integrity if CONFIG.check_tables - analyze_table(PG_DB, logger, "channels", InvidiousChannel) - analyze_table(PG_DB, logger, "channel_videos", ChannelVideo) - analyze_table(PG_DB, logger, "playlists", InvidiousPlaylist) - analyze_table(PG_DB, logger, "playlist_videos", PlaylistVideo) - analyze_table(PG_DB, logger, "nonces", Nonce) - analyze_table(PG_DB, logger, "session_ids", SessionId) - analyze_table(PG_DB, logger, "users", User) - analyze_table(PG_DB, logger, "videos", Video) + check_enum(PG_DB, logger, "privacy", PlaylistPrivacy) + + check_table(PG_DB, logger, "channels", InvidiousChannel) + check_table(PG_DB, logger, "channel_videos", ChannelVideo) + check_table(PG_DB, logger, "playlists", InvidiousPlaylist) + check_table(PG_DB, logger, "playlist_videos", PlaylistVideo) + check_table(PG_DB, logger, "nonces", Nonce) + check_table(PG_DB, logger, "session_ids", SessionId) + check_table(PG_DB, logger, "users", User) + check_table(PG_DB, logger, "videos", Video) if CONFIG.cache_annotations - analyze_table(PG_DB, logger, "annotations", Annotation) + check_table(PG_DB, logger, "annotations", Annotation) end end @@ -386,6 +388,8 @@ get "/watch" do |env| end plid = env.params.query["list"]? + continuation = process_continuation(PG_DB, env.params.query, plid, id) + nojs = env.params.query["nojs"]? nojs ||= "0" @@ -571,7 +575,8 @@ get "/embed/" do |env| if plid = env.params.query["list"]? begin playlist = get_playlist(PG_DB, plid, locale: locale) - videos = get_playlist_videos(PG_DB, playlist, locale: locale) + offset = env.params.query["index"]?.try &.to_i? || 0 + videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) rescue ex error_message = ex.message env.response.status_code = 500 @@ -593,7 +598,9 @@ end get "/embed/:id" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? id = env.params.url["id"] + plid = env.params.query["list"]? + continuation = process_continuation(PG_DB, env.params.query, plid, id) if md = env.params.query["playlist"]? .try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/) @@ -624,7 +631,8 @@ get "/embed/:id" do |env| if plid begin playlist = get_playlist(PG_DB, plid, locale: locale) - videos = get_playlist_videos(PG_DB, playlist, locale: locale) + offset = env.params.query["index"]?.try &.to_i? || 0 + videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) rescue ex error_message = ex.message env.response.status_code = 500 @@ -958,7 +966,7 @@ get "/edit_playlist" do |env| end begin - videos = get_playlist_videos(PG_DB, playlist, page, locale: locale) + videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) rescue ex videos = [] of PlaylistVideo end @@ -1153,13 +1161,31 @@ post "/playlist_ajax" do |env| when "action_edit_playlist" # TODO: Playlist stub when "action_add_video" - # TODO: Wrap error response - raise "Cannot have playlist larger than 1000 videos" if playlist.index.size >= 1000 + if playlist.index.size >= 500 + env.response.status_code = 400 + if redirect + error_message = "Playlist cannot have more than 500 videos" + next templated "error" + else + error_message = {"error" => "Playlist cannot have more than 500 videos"}.to_json + next error_message + end + end video_id = env.params.query["video_id"] - # TODO: Wrap error response - video = get_video(video_id, PG_DB) + begin + video = get_video(video_id, PG_DB) + rescue ex + env.response.status_code = 500 + if redirect + error_message = ex.message + next templated "error" + else + error_message = {"error" => ex.message}.to_json + next error_message + end + end playlist_video = PlaylistVideo.new( title: video.title, @@ -1227,7 +1253,7 @@ get "/playlist" do |env| end begin - videos = get_playlist_videos(PG_DB, playlist, page, locale: locale) + videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) rescue ex videos = [] of PlaylistVideo end @@ -2125,13 +2151,12 @@ post "/watch_ajax" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex + env.response.status_code = 400 if redirect error_message = ex.message - env.response.status_code = 400 next templated "error" else error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 next error_message end end @@ -3144,7 +3169,7 @@ get "/feed/playlist/:plid" do |env| if plid.starts_with? "IV" if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - videos = get_playlist_videos(PG_DB, playlist, locale: locale) + videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) next XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", @@ -4502,14 +4527,15 @@ end env.response.content_type = "application/json" plid = env.params.url["plid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + offset = env.params.query["index"]?.try &.to_i? + offset ||= env.params.query["page"]?.try &.to_i?.try { |page| (page - 1) * 100 } + offset ||= 0 + + continuation = env.params.query["continuation"]? format = env.params.query["format"]? format ||= "json" - continuation = env.params.query["continuation"]? - if plid.starts_with? "RD" next env.redirect "/api/v1/mixes/#{plid}" end @@ -4517,29 +4543,29 @@ end begin playlist = get_playlist(PG_DB, plid, locale) rescue ex - error_message = {"error" => "Playlist is empty"}.to_json - env.response.status_code = 410 + env.response.status_code = 404 + error_message = {"error" => "Playlist does not exist."}.to_json next error_message end - if playlist.privacy == PlaylistPrivacy::Private - user = env.get?("user").try &.as(User) - if !user || user.email != playlist.author - error_message = {"error" => "This playlist is private."}.to_json - env.response.status_code = 403 - next error_message - end + user = env.get?("user").try &.as(User) + if !playlist || !playlist.privacy.public? && playlist.author != user.try &.email + env.response.status_code = 404 + error_message = {"error" => "Playlist does not exist."}.to_json + next error_message end - response = playlist.to_json(locale, config, Kemal.config) + response = playlist.to_json(offset, locale, config, Kemal.config, continuation: continuation) if format == "html" response = JSON.parse(response) playlist_html = template_playlist(response) + index = response["videos"].as_a[1]?.try &.["index"] next_video = response["videos"].as_a[1]?.try &.["videoId"] response = { "playlistHtml" => playlist_html, + "index" => index, "nextVideo" => next_video, }.to_json end @@ -4764,7 +4790,7 @@ get "/api/v1/auth/playlists" do |env| JSON.build do |json| json.array do playlists.each do |playlist| - playlist.to_json(locale, config, Kemal.config, json) + playlist.to_json(0, locale, config, Kemal.config, json) end end end @@ -4796,20 +4822,70 @@ post "/api/v1/auth/playlists" do |env| next templated "error" end - create_playlist(PG_DB, title, privacy, user).to_json(locale, config, Kemal.config) + playlist = create_playlist(PG_DB, title, privacy, user) + playlist.to_json(0, locale, config, Kemal.config) end patch "/api/v1/auth/playlists/:plid" do |env| - # { - # "title" => String?, - # "privacy" => String?, - # "description" => String?, - # } - # TODO: Playlist stub + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.content_type = "application/json" + user = env.get("user").as(User) + + plid = env.params.url["plid"] + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email && !playlist.privacy.public? + env.response.status_code = 404 + error_message = {"error" => "Playlist does not exist."}.to_json + next error_message + end + + if playlist.author != user.email + env.response.status_code = 403 + error_message = {"error" => "Invalid user"}.to_json + next error_message + end + + title = env.params.body["title"]?.try &.as(String).delete("<>") + privacy = env.params.body["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String)) } + description = env.params.body["description"]?.try &.as(String).delete("\r") + + if title != playlist.title || + privacy != playlist.privacy || + description != playlist.description + updated = Time.now + else + updated = playlist.updated + end + + PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + env.response.status_code = 204 end delete "/api/v1/auth/playlists/:plid" do |env| - # TODO: Playlist stub + env.response.content_type = "application/json" + user = env.get("user").as(User) + + plid = env.params.url["plid"] + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email && !playlist.privacy.public? + env.response.status_code = 404 + error_message = {"error" => "Playlist does not exist."}.to_json + next error_message + end + + if playlist.author != user.email + env.response.status_code = 403 + error_message = {"error" => "Invalid user"}.to_json + next error_message + end + + PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) + PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + + env.response.status_code = 204 end get "/api/v1/auth/tokens" do |env| diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 331f63607..bc23f3189 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -596,7 +596,17 @@ def extract_shelf_items(nodeset, ucid = nil, author_name = nil) return items end -def analyze_table(db, logger, table_name, struct_type = nil) +def check_enum(db, logger, enum_name, struct_type = nil) + if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) + logger.puts("CREATE TYPE #{enum_name}") + + db.using_connection do |conn| + conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) + end + end +end + +def check_table(db, logger, table_name, struct_type = nil) # Create table if it doesn't exist begin db.exec("SELECT * FROM #{table_name} LIMIT 0") diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index db0066c90..20f65c2aa 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -35,7 +35,7 @@ struct PlaylistVideo end end - def to_xml(host_url, auto_generated, xml : XML::Builder | Nil = nil) + def to_xml(host_url, auto_generated, xml : XML::Builder? = nil) if xml to_xml(host_url, auto_generated, xml) else @@ -45,7 +45,7 @@ struct PlaylistVideo end end - def to_json(locale, config, kemal_config, json : JSON::Builder, index = nil) + def to_json(locale, config, kemal_config, json : JSON::Builder, index : Int32?) json.object do json.field "title", self.title json.field "videoId", self.id @@ -58,17 +58,23 @@ struct PlaylistVideo generate_thumbnails(json, self.id, config, kemal_config) end - json.field "index", index ? index : self.index + if index + json.field "index", index + json.field "indexId", self.index.to_u64.to_s(16).upcase + else + json.field "index", self.index + end + json.field "lengthSeconds", self.length_seconds end end - def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil, index = nil) + def to_json(locale, config, kemal_config, json : JSON::Builder? = nil, index : Int32? = nil) if json - to_json(locale, config, kemal_config, json, index) + to_json(locale, config, kemal_config, json, index: index) else JSON.build do |json| - to_json(locale, config, kemal_config, json, index) + to_json(locale, config, kemal_config, json, index: index) end end end @@ -87,7 +93,7 @@ struct PlaylistVideo end struct Playlist - def to_json(locale, config, kemal_config, json : JSON::Builder) + def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil) json.object do json.field "type", "playlist" json.field "title", self.title @@ -122,21 +128,21 @@ struct Playlist json.field "videos" do json.array do - videos = get_playlist_videos(PG_DB, self, locale: locale)[0, 5] + videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) videos.each_with_index do |video, index| - video.to_json(locale, config, Kemal.config, json, index) + video.to_json(locale, config, Kemal.config, json) end end end end end - def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil) + def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil) if json - to_json(locale, config, kemal_config, json) + to_json(offset, locale, config, kemal_config, json, continuation: continuation) else JSON.build do |json| - to_json(locale, config, kemal_config, json) + to_json(offset, locale, config, kemal_config, json, continuation: continuation) end end end @@ -166,7 +172,7 @@ enum PlaylistPrivacy end struct InvidiousPlaylist - def to_json(locale, config, kemal_config, json : JSON::Builder) + def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil) json.object do json.field "type", "invidiousPlaylist" json.field "title", self.title @@ -187,21 +193,21 @@ struct InvidiousPlaylist json.field "videos" do json.array do - videos = get_playlist_videos(PG_DB, self, locale: locale)[0, 5] + videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) videos.each_with_index do |video, index| - video.to_json(locale, config, Kemal.config, json, index) + video.to_json(locale, config, Kemal.config, json, offset + index) end end end end end - def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil) + def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil) if json - to_json(locale, config, kemal_config, json) + to_json(offset, locale, config, kemal_config, json, continuation: continuation) else JSON.build do |json| - to_json(locale, config, kemal_config, json) + to_json(offset, locale, config, kemal_config, json, continuation: continuation) end end end @@ -446,38 +452,32 @@ def fetch_playlist(plid, locale) return playlist end -def get_playlist_videos(db, playlist, page = 1, continuation = nil, locale = nil) +def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) if playlist.is_a? InvidiousPlaylist - if continuation - offset = Math.max(0, db.query_one?("SELECT array_position($3, index) - 1 FROM playlist_videos WHERE plid = $1 AND id = $2 ORDER BY array_position($3, index) LIMIT 1", playlist.id, continuation, playlist.index, as: Int32) || 0) - else - offset = (Math.max(page, 1) - 1) * 100 + if !offset + index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", playlist.id, continuation, as: Int64) + offset = playlist.index.index(index) || 0 end - videos = db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", playlist.id, playlist.index, offset, as: PlaylistVideo) - return videos + + db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", playlist.id, playlist.index, offset, as: PlaylistVideo) else - fetch_playlist_videos(playlist.id, page, playlist.video_count, continuation, locale) + fetch_playlist_videos(playlist.id, playlist.video_count, offset, locale, continuation) end end -def fetch_playlist_videos(plid, page, video_count, continuation = nil, locale = nil) +def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil) client = make_client(YT_URL) if continuation html = client.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") html = XML.parse_html(html.body) - index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i? - if index - index -= 1 - end - index ||= 0 - else - index = (page - 1) * 100 + index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i?.try &.- 1 + offset = index || offset end if video_count > 100 - url = produce_playlist_url(plid, index) + url = produce_playlist_url(plid, offset) response = client.get(url) response = JSON.parse(response.body) @@ -487,25 +487,17 @@ def fetch_playlist_videos(plid, page, video_count, continuation = nil, locale = document = XML.parse_html(response["content_html"].as_s) nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])) - videos = extract_playlist(plid, nodeset, index) - else - # Playlist has less than one page of videos, so subsequent pages will be empty - if page > 1 - videos = [] of PlaylistVideo - else - # Extract first page of videos - response = client.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1") - document = XML.parse_html(response.body) - nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])) + videos = extract_playlist(plid, nodeset, offset) + else # Extract first page of videos + response = client.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1") + document = XML.parse_html(response.body) + nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])) - videos = extract_playlist(plid, nodeset, 0) + videos = extract_playlist(plid, nodeset, offset) + end - if continuation - until videos[0].id == continuation - videos.shift - end - end - end + until videos.empty? || videos[0].index == offset + videos.shift end return videos diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ef3f4d4bd..ce8330378 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -1270,6 +1270,20 @@ def itag_to_metadata?(itag : String) return VIDEO_FORMATS[itag]? end +def process_continuation(db, query, plid, id) + continuation = nil + if plid + if index = query["index"]?.try &.to_i? + continuation = index + else + continuation = id + end + continuation ||= 0 + end + + continuation +end + def process_video_params(query, preferences) annotations = query["iv_load_policy"]?.try &.to_i? autoplay = query["autoplay"]?.try &.to_i? diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr index 1a253026b..6c06bf2ed 100644 --- a/src/invidious/views/embed.ecr +++ b/src/invidious/views/embed.ecr @@ -29,6 +29,7 @@