Skip to content

Commit

Permalink
Clean up data import/export
Browse files Browse the repository at this point in the history
  • Loading branch information
omarroth committed Nov 9, 2018
1 parent 6066615 commit b77c73d
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 69 deletions.
1 change: 0 additions & 1 deletion shard.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ dependencies:
github: detectlanguage/detectlanguage-crystal
kemal:
github: kemalcr/kemal
commit: b389022
pg:
github: will/crystal-pg

Expand Down
151 changes: 83 additions & 68 deletions src/invidious.cr
Original file line number Diff line number Diff line change
Expand Up @@ -1230,114 +1230,115 @@ post "/data_control" do |env|
case part.name
when "import_invidious"
body = JSON.parse(body)
body["subscriptions"]?.try &.as_a.each do |ucid|
ucid = ucid.as_s

if !user.subscriptions.includes? ucid
if body["subscriptions"]?
user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s }
user.subscriptions.uniq!

user.subscriptions.each do |ucid|
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)

PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
user.subscriptions << ucid
rescue ex
next
end
end

PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
end

if body["watch_history"]?
watched = user.watched + body["watch_history"].as_a.map { |a| a.as_s }
watched.uniq!
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", watched, user.email)
user.watched += body["watch_history"].as_a.map { |a| a.as_s }
user.watched.uniq!
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
end

if body["preferences"]?
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", body["preferences"].to_json, user.email)
user.preferences = Preferences.from_json(body["preferences"].to_json)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email)
end
when "import_youtube"
subscriptions = XML.parse(body)
subscriptions.xpath_nodes(%q(//outline[@type="rss"])).each do |channel|
ucid = channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]

if !user.subscriptions.includes? ucid
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)

PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
user.subscriptions << ucid
rescue ex
next
end
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
user.subscriptions.uniq!

user.subscriptions.each do |ucid|
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)
rescue ex
next
end
end
when "import_freetube"
body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).each do |md|
ucid = md["channel_id"]

if !user.subscriptions.includes? ucid
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)

PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
user.subscriptions << ucid
rescue ex
next
end
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
when "import_freetube"
user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md|
md["channel_id"]
end
user.subscriptions.uniq!

user.subscriptions.each do |ucid|
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)
rescue ex
next
end
end

PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
when "import_newpipe_subscriptions"
body = JSON.parse(body)
body["subscriptions"].as_a.each do |channel|
ucid = channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]

if !user.subscriptions.includes? ucid
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)

PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
user.subscriptions << ucid
rescue ex
next
end
user.subscriptions += body["subscriptions"].as_a.map do |channel|
channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
user.subscriptions.uniq!

user.subscriptions.each do |ucid|
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)
rescue ex
next
end
end

PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
when "import_newpipe"
Zip::Reader.open(body) do |file|
Zip::Reader.open(IO::Memory.new(body)) do |file|
file.each_entry do |entry|
if entry.filename == "newpipe.db"
# We do this because the SQLite driver cannot parse a database from an IO
# Currently: channel URLs can **only** be subscriptions, and
# video URLs can **only** be watch history, so this works okay for now.

db = entry.io.gets_to_end
db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/) do |md|
id = md["id"]

if !user.watched.includes? id
PG_DB.exec("UPDATE users SET watched = array_append(watched,$1) WHERE email = $2", id, user.email)
user.watched << id
end
user.watched += db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/).map do |md|
md["id"]
end
user.watched.uniq!

db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/) do |md|
ucid = md["ucid"]
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)

if !user.subscriptions.includes? ucid
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)

PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
user.subscriptions << ucid
rescue ex
next
end
user.subscriptions += db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/).map do |md|
md["ucid"]
end
user.subscriptions.uniq!

user.subscriptions.each do |ucid|
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)
rescue ex
next
end
end

PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
end
end
end
Expand Down Expand Up @@ -1629,6 +1630,20 @@ get "/feed/subscriptions" do |env|
end
end

get "/feed/history" do |env|
user = env.get? "user"
referer = get_referer(env)

if user
user = user.as(User)
watched = user.watched.reverse

templated "history"
else
env.redirect referer
end
end

get "/feed/channel/:ucid" do |env|
env.response.content_type = "text/xml"
ucid = env.params.url["ucid"]
Expand Down

0 comments on commit b77c73d

Please sign in to comment.