diff --git a/.gitignore b/.gitignore index 7a26e1a6d..686589ae2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ /invidious /sentry /config/config.yml +/invidious-videojs-dep-install \ No newline at end of file diff --git a/scripts/fetch-player-dependencies.cr b/scripts/fetch-player-dependencies.cr index 813e4ce4e..097d61a7f 100755 --- a/scripts/fetch-player-dependencies.cr +++ b/scripts/fetch-player-dependencies.cr @@ -1,145 +1,277 @@ require "http" require "yaml" +require "file_utils" require "digest/sha1" require "option_parser" require "colorize" -# Taken from https://crystal-lang.org/api/1.1.1/OptionParser.html -minified = false -OptionParser.parse do |parser| - parser.banner = "Usage: Fetch VideoJS dependencies [arguments]" - parser.on("-m", "--minified", "Use minified versions of VideoJS dependencies (performance and bandwidth benefit)") { minified = true } - - parser.on("-h", "--help", "Show this help") do - puts parser - exit - end +# Represents an "install_instruction" section specified per dependency in `videojs-dependencies.yml` +# +# This is used to modify the download logic for dependencies that are packaged differently. +struct InstallInstruction + include YAML::Serializable - parser.invalid_option do |flag| - STDERR.puts "ERROR: #{flag} is not a valid option." - STDERR.puts parser - exit(1) - end + property js_path : String? = nil + property css_path : String? = nil + property download_as : String? = nil + property no_styling : Bool = false end -required_dependencies = File.open("videojs-dependencies.yml") do |file| - YAML.parse(file).as_h -end +# Object representing a dependency specified within `videojs-dependencies.yml` +class ConfigDependency + include YAML::Serializable -def update_versions_yaml(required_dependencies, minified, dep_name) - File.open("assets/videojs/#{dep_name}/versions.yml", "w") do |io| - YAML.build(io) do |builder| - builder.mapping do - # Versions - builder.scalar "version" - builder.scalar "#{required_dependencies[dep_name]["version"]}" + property version : String + property shasum : String - builder.scalar "minified" - builder.scalar minified - end + property install_instructions : InstallInstruction? = nil + + # Checks if the current dependency needs to be installed/updated + def fetch?(name : String) + path = "assets/videojs/#{name}" + + # Check for missing dependency files + # + # Does the directory exist? + # Does the Javascript file exist? + # Does the CSS file exist? + if !Dir.exists?(path) + Dir.mkdir(path) + return true + elsif !(File.exists?("#{path}/#{name}.js") || File.exists?("#{path}/versions.yml")) + return true + elsif !(self.install_instructions.try &.no_styling) && !File.exists?("#{path}/#{name}.css") + return true end - end -end -# The first step is to check which dependencies we'll need to install. -# If the version we have requested in `videojs-dependencies.yml` is the -# same as what we've installed, we shouldn't do anything. Likewise, if it's -# different or the requested dependency just isn't present, then it needs to be -# installed. - -# Since we can't know when videojs-youtube-annotations is updated, we'll just always fetch -# a new copy each time. -dependencies_to_install = [] of String - -required_dependencies.keys.each do |dep| - dep = dep.to_s - path = "assets/videojs/#{dep}" - # Check for missing dependencies - if !Dir.exists?(path) - Dir.mkdir(path) - dependencies_to_install << dep - else - config = File.open("#{path}/versions.yml") do |file| + # Check if we need to update the dependency + + versions = File.open("#{path}/versions.yml") do |file| YAML.parse(file).as_h end - if config["version"].as_s != required_dependencies[dep]["version"].as_s || config["minified"].as_bool != minified - `rm -rf #{path}/*.js #{path}/*.css` - dependencies_to_install << dep + if versions["version"].as_s != self.version || versions["minified"].as_bool != CONFIG.minified + # Clear directory + {"*.js", "*.css"}.each do |file_types| + Dir.glob("#{path}/#{file_types}").each do |file_path| + File.delete(file_path) + end + end + + return true end + + return false end end -# Now we begin the fun part of installing the dependencies. -# But first we'll setup a temp directory to store the plugins -tmp_dir_path = "#{Dir.tempdir}/invidious-videojs-dep-install" -Dir.mkdir(tmp_dir_path) if !Dir.exists? tmp_dir_path +# Object representing the `videojs-dependencies.yml` file +class PlayerDependenciesConfig + include YAML::Serializable -channel = Channel(String | Exception).new + property version : String + property registry_url : String + property cache_directory : String + property dependencies : Hash(YAML::Any, ConfigDependency) -dependencies_to_install.each do |dep| - spawn do - dep_name = dep - download_path = "#{tmp_dir_path}/#{dep}" - dest_path = "assets/videojs/#{dep}" + def get_dependencies_to_fetch + return self.dependencies.select { |name, config| config.fetch?(name.as_s) } + end +end - HTTP::Client.get("https://registry.npmjs.org/#{dep}/-/#{dep}-#{required_dependencies[dep]["version"]}.tgz") do |response| - Dir.mkdir(download_path) - data = response.body_io.gets_to_end - File.write("#{download_path}/package.tgz", data) +# Runtime Dependency config for easy access to all the variables +class Config + property minified : Bool + property skip_checksum : Bool + property clear_cache : Bool + + property dependency_config : PlayerDependenciesConfig + + def initialize(path : String) + @minified = false + @skip_checksum = false + @clear_cache = false - # https://github.com/iv-org/invidious/pull/2397#issuecomment-922375908 - if `sha1sum #{download_path}/package.tgz`.split(" ")[0] != required_dependencies[dep]["shasum"] - raise Exception.new("Checksum for '#{dep}' failed") + @dependency_config = PlayerDependenciesConfig.from_yaml(File.read(path)) + end + + # Less verbose way to access @dependency_config.registry_url + def registry_url + return @dependency_config.registry_url + end + + # Less verbose way to access @dependency_config.cache_directory + def cache_directory + return @dependency_config.cache_directory + end +end + +# Object representing a player dependency +class Dependency + @config : ConfigDependency + + def initialize(@config : ConfigDependency, @dependency : String) + @download_path = "#{CONFIG.cache_directory}/#{@dependency}" + @destination_path = "assets/videojs/#{@dependency}" + end + + private def validate_checksum(io) + return if CONFIG.skip_checksum + + digest = Digest::SHA1.hexdigest(io) + if digest != @config.shasum + raise IO::Error.new("Checksum for '#{@dependency}' failed. \"#{digest}\" does not match configured \"#{@config.shasum}\"") + end + end + + # Requests and downloads a specific dependency from NPM + # + # Validates a cached tarball if it already exists. + private def request_dependency + downloaded_package_path = "#{@download_path}/package.tgz" + + # Create a download directory for the dependency if it does not already exist + if Dir.exists?(@download_path) + # Validate checksum of existing cached tarball + # Fetches a new one when the checksum fails. + if File.exists?(downloaded_package_path) + begin + return self.validate_checksum(File.open(downloaded_package_path)) + rescue IO::Error + end end + else + Dir.mkdir(@download_path) end - # Unless we install an external dependency, crystal provides no way of extracting a tarball. - # Thus we'll go ahead and call a system command. - `tar -vzxf '#{download_path}/package.tgz' -C '#{download_path}'` - raise "Extraction for #{dep} failed" if !$?.success? + HTTP::Client.get("#{CONFIG.registry_url}/#{@dependency}/-/#{@dependency}-#{@config.version}.tgz") do |response| + data = response.body_io.gets_to_end + File.write(downloaded_package_path, data) + self.validate_checksum(data) + end + end + + # Moves a VideoJS dependency file of the given extension from extracted tarball to Invidious directory + private def move_file(full_target_path, extension) + minified_target_path = sprintf(full_target_path, {"file_extension": ".min.#{extension}"}) + + if CONFIG.minified && File.exists?(minified_target_path) + target_path = minified_target_path + else + target_path = sprintf(full_target_path, {"file_extension": ".#{extension}"}) + end - # Would use File.rename in the following steps but for some reason it just doesn't work here. - # Video.js itself is structured slightly differently - dep = "video" if dep == "video.js" + if download_as = @config.install_instructions.try &.download_as + destination_path = "#{@destination_path}/#{sprintf(download_as, {"file_extension": ".#{extension}"})}" + else + destination_path = @destination_path + end - # This dep nests everything under an additional JS or CSS folder - if dep == "silvermine-videojs-quality-selector" - js_path = "js/" + FileUtils.cp(target_path, destination_path) + end - # It also stores their quality selector as `quality-selector.css` - `mv #{download_path}/package/dist/css/quality-selector.css #{dest_path}/quality-selector.css` + # Fetch path of where a VideoJS dependency is located in the extracted tarball + private def fetch_path(is_css) + if is_css + raw_target_path = @config.install_instructions.try &.css_path else - js_path = "" + raw_target_path = @config.install_instructions.try &.js_path end - # Would use File.rename but for some reason it just doesn't work here. - if minified && File.exists?("#{download_path}/package/dist/#{js_path}#{dep}.min.js") - `mv #{download_path}/package/dist/#{js_path}#{dep}.min.js #{dest_path}/#{dep}.js` + if raw_target_path + return "#{@download_path}/package/#{raw_target_path}" else - `mv #{download_path}/package/dist/#{js_path}#{dep}.js #{dest_path}/#{dep}.js` + return "#{@download_path}/package/dist/#{@dependency}%{file_extension}" end + end - # Fetch CSS which isn't guaranteed to exist - # - # Also, video JS changes structure here once again... - dep = "video-js" if dep == "video" + # Wrapper around `#move_file` to move the dependency's JS file + private def move_js_file + return self.move_file(self.fetch_path(is_css: false), "js") + end - # VideoJS marker uses a dot on the CSS files. - dep = "videojs.markers" if dep == "videojs-markers" + # Wrapper around `#move_file` to move the dependency's CSS file + # + # Does nothing with the CSS file does not exist. + private def move_css_file + path = self.fetch_path(is_css: true) - if File.exists?("#{download_path}/package/dist/#{dep}.css") - if minified && File.exists?("#{download_path}/package/dist/#{dep}.min.css") - `mv #{download_path}/package/dist/#{dep}.min.css #{dest_path}/#{dep}.css` - else - `mv #{download_path}/package/dist/#{dep}.css #{dest_path}/#{dep}.css` + if File.exists?(sprintf(path, {"file_extension": ".css"})) + return move_file(path, "css") + end + end + + # Updates the dependency's versions.yml with the current fetched version and its minified status + private def update_versions_yaml + File.open("#{@destination_path}/versions.yml", "w") do |io| + YAML.build(io) do |builder| + builder.mapping do + # Versions + builder.scalar "version" + builder.scalar "#{@config.version}" + + builder.scalar "minified" + builder.scalar CONFIG.minified + end end end + end + + # Installs a VideoJS dependency into Invidious + def install + self.request_dependency + + # Crystal's stdlib provides no way of extracting a tarball + `tar -vzxf '#{@download_path}/package.tgz' -C '#{@download_path}'` + raise "Extraction for #{@dependency} failed" if !$?.success? + + self.move_js_file + self.move_css_file + + self.update_versions_yaml + end +end - # Update/create versions file for the dependency - update_versions_yaml(required_dependencies, minified, dep_name) +CONFIG = Config.new("videojs-dependencies.yml") - channel.send(dep_name) +# Hacky solution to get separated arguments when called from invidious.cr +if ARGV.size == 1 + parser_args = [] of String + ARGV[0].split(",") { |str| parser_args << str.strip } +else + parser_args = ARGV +end + +# Taken from https://crystal-lang.org/api/1.1.1/OptionParser.html +OptionParser.parse(parser_args) do |parser| + parser.banner = "Usage: Fetch VideoJS dependencies [arguments]" + parser.on("-m", "--minified", "Use minified versions of VideoJS dependencies (performance and bandwidth benefit)") { CONFIG.minified = true } + parser.on("--skip-checksum", "Skips the checksum validation of downloaded files") { CONFIG.skip_checksum = true } + parser.on("--clear-cache", "Clears the cache and re-downloads all dependency files") { CONFIG.clear_cache = true } + + parser.on("-h", "--help", "Show this help") do + puts parser + exit + end + + parser.invalid_option do |flag| + STDERR.puts "ERROR: #{flag} is not a valid option." + STDERR.puts parser + exit(1) + end +end + +# Create cache directory +Dir.mkdir(CONFIG.cache_directory) if !Dir.exists? CONFIG.cache_directory + +dependencies_to_install = CONFIG.dependency_config.get_dependencies_to_fetch +channel = Channel(String | Exception).new + +dependencies_to_install.each do |dep_name, dependency_config| + spawn do + dependency = Dependency.new(dependency_config, dep_name.as_s) + dependency.install + channel.send(dep_name.as_s) rescue ex channel.send(ex) end @@ -161,4 +293,6 @@ else end # Cleanup -`rm -rf #{tmp_dir_path}` +if CONFIG.clear_cache + FileUtils.rm_r("#{CONFIG.cache_directory}") +end diff --git a/src/invidious.cr b/src/invidious.cr index c8cac80ec..3e02dde33 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -144,12 +144,26 @@ Invidious::Database.check_integrity(CONFIG) # Running the script by itself would show some colorful feedback while this doesn't. # Perhaps we should just move the script to runtime in order to get that feedback? - {% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %} - {% if flag?(:minified_player_dependencies) %} - {% puts run("../scripts/fetch-player-dependencies.cr", "--minified").stringify %} - {% else %} - {% puts run("../scripts/fetch-player-dependencies.cr").stringify %} + {% fetch_script_arguments = [] of StringLiteral %} + + {% + potential_arguments = { + {:minified_player_dependencies, "--minified"}, + {:skip_player_dependencies_checksum, "--skip-checksum"}, + {:clear_player_dependencies_cache, "--clear-cache"}, + } + %} + + {% for potential_argument in potential_arguments %} + {% flag_, script_argument = potential_argument %} + + {% if flag?(flag_) %} + {% fetch_script_arguments << script_argument.id %} + {% end %} {% end %} + + {% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %} + {% puts run("../scripts/fetch-player-dependencies.cr", fetch_script_arguments.splat).stringify %} {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} {% end %} diff --git a/src/invidious/views/components/player_sources.ecr b/src/invidious/views/components/player_sources.ecr index 9af3899c4..4d4543f40 100644 --- a/src/invidious/views/components/player_sources.ecr +++ b/src/invidious/views/components/player_sources.ecr @@ -1,12 +1,12 @@ - + - + - + diff --git a/videojs-dependencies.yml b/videojs-dependencies.yml index e9ccc9dde..38f4cff38 100644 --- a/videojs-dependencies.yml +++ b/videojs-dependencies.yml @@ -1,54 +1,80 @@ -# Due to a 'video append of' error (see #3011), we're stuck on 7.12.1. -video.js: - version: 7.12.1 - shasum: 1d12eeb1f52e3679e8e4c987d9b9eb37e2247fa2 - -videojs-contrib-quality-levels: - version: 2.1.0 - shasum: 046e9e21ed01043f512b83a1916001d552457083 - -videojs-http-source-selector: - version: 1.1.6 - shasum: 073aadbea0106ba6c98d6b611094dbf8554ffa1f - -videojs-markers: - version: 1.0.1 - shasum: d7f8d804253fd587813271f8db308a22b9f7df34 - -videojs-mobile-ui: - version: 0.6.1 - shasum: 0e146c4c481cbee0729cb5e162e558b455562cd0 - -videojs-overlay: - version: 2.1.4 - shasum: 5a103b25374dbb753eb87960d8360c2e8f39cc05 - -videojs-share: - version: 3.2.1 - shasum: 0a3024b981387b9d21c058c829760a72c14b8ceb - -videojs-vr: - version: 1.8.0 - shasum: 7f2f07f760d8a329c615acd316e49da6ee8edd34 - -videojs-vtt-thumbnails: - version: 0.0.13 - shasum: d1e7d47f4ed80bb52f5fc4f4bad4bfc871f5970f - -# We're using iv-org's fork of videojs-quality-selector, -# which isn't published on NPM, and doesn't have any -# easy way of fetching the compiled variant. -# -# silvermine-videojs-quality-selector: -# version: 1.1.2 -# shasum: 94033ff9ee52ba6da1263b97c9a74d5b3dfdf711 - - -# Ditto. Although this extension contains the complied variant in its git repo, -# it lacks any sort of versioning. As such, the script will ignore it. -# -# videojs-youtube-annotations: -# github: https://github.com/afrmtbl/videojs-youtube-annotations +version: 1 +registry_url: "https://registry.npmjs.org" + +# Dependencies are stored as /package.tgz +cache_directory: "./invidious-videojs-dep-install" + +dependencies: + #Due to a 'video append of' error (see #3011), we're stuck on 7.12.1. + video.js: + version: 7.12.1 + shasum: 1d12eeb1f52e3679e8e4c987d9b9eb37e2247fa2 + + install_instructions: + js_path: "dist/video%{file_extension}" + css_path: "dist/video-js%{file_extension}" + + # Normalize names to simplify File.exists? check + download_as: "video.js%{file_extension}" + + videojs-contrib-quality-levels: + version: 2.1.0 + shasum: 046e9e21ed01043f512b83a1916001d552457083 + + install_instructions: + no_styling: true + + videojs-http-source-selector: + version: 1.1.6 + shasum: 073aadbea0106ba6c98d6b611094dbf8554ffa1f + + videojs-markers: + version: 1.0.1 + shasum: d7f8d804253fd587813271f8db308a22b9f7df34 + + install_instructions: + css_path: "dist/videojs.markers%{file_extension}" + download_as: "videojs-markers%{file_extension}" + + videojs-mobile-ui: + version: 0.6.1 + shasum: 0e146c4c481cbee0729cb5e162e558b455562cd0 + + videojs-overlay: + version: 2.1.4 + shasum: 5a103b25374dbb753eb87960d8360c2e8f39cc05 + + videojs-share: + version: 3.2.1 + shasum: 0a3024b981387b9d21c058c829760a72c14b8ceb + + videojs-vr: + version: 1.8.0 + shasum: 7f2f07f760d8a329c615acd316e49da6ee8edd34 + + videojs-vtt-thumbnails: + version: 0.0.13 + shasum: d1e7d47f4ed80bb52f5fc4f4bad4bfc871f5970f + + # We're using iv-org's fork of videojs-quality-selector, + # which isn't published on NPM, and doesn't have any + # easy way of fetching the compiled variant. + + # silvermine-videojs-quality-selector: + # version: 1.1.2 + # shasum: 94033ff9ee52ba6da1263b97c9a74d5b3dfdf711 + + # install_instructions: + # js_path: "dist/js/silvermine-videojs-quality-selector%{file_extension}" + # css_path: "dist/css/quality-selector%{file_extension}" + # download_as: silvermine-videojs-quality-selector%{file_extension} + + + # Ditto. Although this extension contains the complied variant in its git repo, + # it lacks any sort of versioning. As such, the script will ignore it. + # + # videojs-youtube-annotations: + # github: https://github.com/afrmtbl/videojs-youtube-annotations