storage manager: remove hierarchical option.

Remove the `hierarchical` file storage option. This means that image
files are always stored in MD5-based subdirectories, like this:

   https://danbooru.donmai.us/data/original/f3/a7/f3a70a89c350b5ed4db22dbb25b934bb.jpg
   https://danbooru.donmai.us/data/sample/f3/a7/sample-f3a70a89c350b5ed4db22dbb25b934bb.jpg
   https://danbooru.donmai.us/data/preview/f3/a7/f3a70a89c350b5ed4db22dbb25b934bb.jpg

instead of in a single flat directory, like this:

   https://danbooru.donmai.us/data/original/f3a70a89c350b5ed4db22dbb25b934bb.jpg

This option is removed because storing files in a single directory is a
bad idea for large installations, and migrating from a single directory
to subdirectories later is a pain.

Downstream boorus who still have files in the old layout can migrate by
running this script:

   `./script/fixes/077_symlink_subdirectories.rb`

This will create symlinks that redirect the 00-ff subdirectories back to
the current directory, so that you can still store files in a single
directory, but use URLs containing subdirectories.

You should also make sure to remove the `hierarchical` option from
`storage_manager` in `config/danbooru_local_config.rb` if you set it
there.

app/logical/storage_manager.rb

@@ -1,12 +1,11 @@
 class StorageManager
   class Error < StandardError; end
 
-  attr_reader :base_url, :base_dir, :hierarchical, :tagged_filenames
+  attr_reader :base_url, :base_dir, :tagged_filenames
 
-  def initialize(base_url:, base_dir:, hierarchical: false, tagged_filenames: Danbooru.config.enable_seo_post_urls)
+  def initialize(base_url:, base_dir:, tagged_filenames: Danbooru.config.enable_seo_post_urls)
     @base_url = base_url.chomp("/")
     @base_dir = base_dir
-    @hierarchical = hierarchical
     @tagged_filenames = tagged_filenames
   end
 
@@ -98,11 +97,7 @@ def file_name(md5, file_ext, type)
   end
 
   def subdir_for(md5)
-    if hierarchical
-      "#{md5[0..1]}/#{md5[2..3]}/"
-    else
-      ""
-    end
+    "#{md5[0..1]}/#{md5[2..3]}/"
   end
 
   def seo_tags(post)

app/logical/storage_manager/match.rb

@@ -8,15 +8,15 @@
 #
 #   StorageManager::Match.new do |matcher|
 #     matcher.add_manager(type: :crop) do
-#       StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", hierarchical: true, base_dir: "/var/www/raikou3")
+#       StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", base_dir: "/var/www/raikou3")
 #     end
 #
 #     matcher.add_manager(id: 1..850_000) do
-#       StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", hierarchical: true, base_dir: "/var/www/raikou1")
+#       StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/var/www/raikou1")
 #     end
 #
 #     matcher.add_manager(id: 850_001..2_000_000) do
-#       StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", hierarchical: true, base_dir: "/var/www/raikou2")
+#       StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/var/www/raikou2")
 #     end
 #
 #     matcher.add_manager(id: 1..3_000_000, type: [:large, :original]) do

config/danbooru_default_config.rb

@@ -33,7 +33,7 @@ def hostname
     # you don't support HTTPS. Protip: use ngrok.com for easy HTTPS support
     # during development.
     def canonical_url
-      "https://#{Danborou.config.hostname}"
+      "https://#{Danbooru.config.hostname}"
     end
 
     # Contact email address of the admin.
@@ -152,22 +152,20 @@ def storage_manager
       # Store files on the local filesystem.
       # base_dir - where to store files (default: under public/data)
       # base_url - where to serve files from (default: https://#{hostname}/data)
-      # hierarchical: false - store files in a single directory
-      # hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash
-      StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data"), hierarchical: false)
+      StorageManager::Local.new(base_url: "#{Danbooru.config.canonical_url}/data", base_dir: Rails.root.join("public/data"))
 
       # Store files on one or more remote host(s). Configure SSH settings in
       # ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
-      # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {})
+      # StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", ssh_options: {})
 
       # Select the storage method based on the post's id and type (preview, large, or original).
       # StorageManager::Hybrid.new do |id, md5, file_ext, type|
       #   ssh_options = { user: "danbooru" }
       #
       #   if type.in?([:large, :original]) && id.in?(0..850_000)
-      #     StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
+      #     StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options)
       #   elsif type.in?([:large, :original]) && id.in?(850_001..2_000_000)
-      #     StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
+      #     StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", ssh_options: ssh_options)
       #   elsif type.in?([:large, :original]) && id.in?(2_000_001..3_000_000)
       #     StorageManager::SFTP.new(*all_server_hosts, base_url: "https://hijiribe.donmai.us/data", ssh_options: ssh_options)
       #   else
@@ -182,7 +180,7 @@ def backup_storage_manager
       StorageManager::Null.new
 
       # Backup files to /mnt/backup on the local filesystem.
-      # StorageManager::Local.new(base_dir: "/mnt/backup", hierarchical: false)
+      # StorageManager::Local.new(base_dir: "/mnt/backup")
 
       # Backup files to /mnt/backup on a remote system. Configure SSH settings
       # in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)

script/fixes/077_symlink_subdirectories.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require_relative "../../config/environment"
+
+def create_symlinks(dir)
+  FileUtils.mkdir_p(dir)
+
+  (0..255).each do |i|
+    subdir = "#{dir}/#{"%.2x" % i}"
+
+    if File.exist?(subdir)
+      puts "skipping #{subdir}"
+    else
+      puts "ln -sf . #{subdir}"
+      FileUtils.ln_sf(".", subdir)
+    end
+  end
+end
+
+root = Rails.root.join("public/data")
+
+create_symlinks(root)
+create_symlinks("#{root}/sample")
+create_symlinks("#{root}/preview")
+create_symlinks("#{root}/crop")
+
+FileUtils.ln_sf(".", "#{root}/original") unless File.exist?("#{root}/original")

test/test_helper.rb

@@ -49,9 +49,9 @@ class ActiveSupport::TestCase
     Socket.stubs(:gethostname).returns("www.example.com")
 
     @temp_dir = Dir.mktmpdir("danbooru-temp-")
-    storage_manager = StorageManager::Local.new(base_dir: @temp_dir)
+    storage_manager = StorageManager::Local.new(base_url: "https://www.example.com/data", base_dir: @temp_dir)
     Danbooru.config.stubs(:storage_manager).returns(storage_manager)
-    Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new)
+    Danbooru.config.stubs(:backup_storage_manager).returns(StorageManager::Null.new(base_url: "/", base_dir: "/"))
   end
 
   teardown do

test/unit/post_test.rb

@@ -1972,14 +1972,14 @@ def teardown
 
   context "URLs:" do
     should "generate the correct urls for animated gifs" do
-      manager = StorageManager::Local.new(base_url: "https://test.com/data")
+      manager = StorageManager::Local.new(base_url: "https://test.com/data", base_dir: "/")
       Danbooru.config.stubs(:storage_manager).returns(manager)
 
       @post = build(:post, md5: "deadbeef", file_ext: "gif", tag_string: "animated_gif")
 
-      assert_equal("https://test.com/data/preview/deadbeef.jpg", @post.preview_file_url)
-      assert_equal("https://test.com/data/deadbeef.gif", @post.large_file_url)
-      assert_equal("https://test.com/data/deadbeef.gif", @post.file_url)
+      assert_equal("https://test.com/data/preview/de/ad/deadbeef.jpg", @post.preview_file_url)
+      assert_equal("https://test.com/data/original/de/ad/deadbeef.gif", @post.large_file_url)
+      assert_equal("https://test.com/data/original/de/ad/deadbeef.gif", @post.file_url)
     end
   end
 

test/unit/storage_manager_test.rb

@@ -81,10 +81,11 @@ class StorageManagerTest < ActiveSupport::TestCase
         @storage_manager.store_file(StringIO.new("data"), @post, :preview)
         @storage_manager.store_file(StringIO.new("data"), @post, :large)
         @storage_manager.store_file(StringIO.new("data"), @post, :original)
+        subdir = "#{@post.md5[0..1]}/#{@post.md5[2..3]}"
 
-        @file_path = "#{@temp_dir}/preview/#{@post.md5}.jpg"
-        @large_file_path = "#{@temp_dir}/sample/sample-#{@post.md5}.jpg"
-        @preview_file_path = "#{@temp_dir}/#{@post.md5}.#{@post.file_ext}"
+        @file_path = "#{@temp_dir}/preview/#{subdir}/#{@post.md5}.jpg"
+        @large_file_path = "#{@temp_dir}/sample/#{subdir}/sample-#{@post.md5}.jpg"
+        @preview_file_path = "#{@temp_dir}/original/#{subdir}/#{@post.md5}.#{@post.file_ext}"
       end
 
       should "store the files at the correct path" do
@@ -108,10 +109,11 @@ class StorageManagerTest < ActiveSupport::TestCase
       should "return the correct urls" do
         @post = FactoryBot.create(:post, file_ext: "png")
         @storage_manager.stubs(:tagged_filenames).returns(false)
+        subdir = "#{@post.md5[0..1]}/#{@post.md5[2..3]}"
 
-        assert_equal("/data/#{@post.md5}.png", @storage_manager.file_url(@post, :original))
-        assert_equal("/data/sample/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large))
-        assert_equal("/data/preview/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview))
+        assert_equal("/data/original/#{subdir}/#{@post.md5}.png", @storage_manager.file_url(@post, :original))
+        assert_equal("/data/sample/#{subdir}/sample-#{@post.md5}.jpg", @storage_manager.file_url(@post, :large))
+        assert_equal("/data/preview/#{subdir}/#{@post.md5}.jpg", @storage_manager.file_url(@post, :preview))
       end
 
       should "return the correct url for flash files" do
@@ -145,15 +147,15 @@ class StorageManagerTest < ActiveSupport::TestCase
         @storage_manager.store_file(StringIO.new("post1"), @post1, :original)
         @storage_manager.store_file(StringIO.new("post2"), @post2, :original)
 
-        assert(File.exist?("#{@temp_dir}/i1/#{@post1.md5}.png"))
-        assert(File.exist?("#{@temp_dir}/i2/#{@post2.md5}.png"))
+        assert(File.exist?("#{@temp_dir}/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png"))
+        assert(File.exist?("#{@temp_dir}/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png"))
       end
     end
 
     context "#file_url method" do
       should "generate /i1 urls for odd posts and /i2 urls for even posts" do
-        assert_equal("/i1/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
-        assert_equal("/i2/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
+        assert_equal("/i1/original/#{@post1.md5[0..1]}/#{@post1.md5[2..3]}/#{@post1.md5}.png", @storage_manager.file_url(@post1, :original))
+        assert_equal("/i2/original/#{@post2.md5[0..1]}/#{@post2.md5[2..3]}/#{@post2.md5}.png", @storage_manager.file_url(@post2, :original))
       end
     end
   end