アップロードのリマッピングが少し熱心すぎる

RGJ · 2019 年 5 月 28 日午後 3:44

When a backup is restored to a database with a different name, there is a neat little piece of code in restorer.rb that attempts to fix this.

github.com/discourse/discourse

lib/backup_restore/restorer.rb

main


      
          # frozen_string_literal: true
          
          require "colored2"
          
          module BackupRestore
            RestoreDisabledError = Class.new(RuntimeError)
            FilenameMissingError = Class.new(RuntimeError)
          
            class Restorer
              delegate :log, to: :@logger, private: true
          
              attr_reader :success
          
              def initialize(
                user_id:,
                filename:,
                factory:,
                disable_emails: true,
                location: nil,
                interactive: false
              )
                @user_id = user_id
                @filename = filename
                @factory = factory
                @logger = factory.logger
                @disable_emails = disable_emails
                @interactive = interactive
          
                ensure_restore_is_enabled
                ensure_we_have_a_user
                ensure_we_have_a_filename
          
                @success = false
                @current_db = RailsMultisite::ConnectionManagement.current_db
          
                @system = factory.create_system_interface
                @backup_file_handler = factory.create_backup_file_handler(@filename, @current_db, location)
                @database_restorer = factory.create_database_restorer(@current_db)
                @uploads_restorer = factory.create_uploads_restorer
              end
          
              def run
                log "[STARTED]"
                log "'#{@user_info[:username]}' has started the restore!"
          
                # FIXME not atomic!
                ensure_no_operation_is_running
                @system.mark_restore_as_running
          
                @system.listen_for_shutdown_signal
          
                @filename, @tmp_directory, db_dump_path = @backup_file_handler.decompress
                validate_backup_metadata
          
                @system.enable_readonly_mode
                @system.pause_sidekiq("restore")
                @system.wait_for_sidekiq
                @system.flush_redis
                @system.clear_sidekiq_queues
          
                @database_restorer.restore(db_dump_path, @interactive)
          
                reload_site_settings
                run_seed_fu
          
                @system.disable_readonly_mode
          
                clear_category_cache
                clear_stats
                reload_translations
          
                restore_uploads
          
                clear_emoji_cache
                clear_theme_cache
          
                after_restore_hook
              rescue Compression::Strategy::ExtractFailed
                log "ERROR: The uncompressed file is too big. Consider increasing the hidden " \
                      '"decompressed_backup_max_file_size_mb" setting.'
                @database_restorer.rollback
              rescue SystemExit
                log "Restore process was cancelled!"
                @database_restorer.rollback
              rescue => ex
                log "EXCEPTION: " + ex.message
                log ex.backtrace.join("\n")
                @database_restorer.rollback
              else
                @success = true
              ensure
                clean_up
                notify_user
                log "Finished!"
          
                @success ? log("[SUCCESS]") : log("[FAILED]")
              end
          
              protected
          
              def ensure_restore_is_enabled
                return if Rails.env.development? || SiteSetting.allow_restore?
                raise BackupRestore::RestoreDisabledError
              end
          
              def ensure_no_operation_is_running
                raise BackupRestore::OperationRunningError if BackupRestore.is_operation_running?
              end
          
              def ensure_we_have_a_user
                user = User.find_by(id: @user_id)
                raise Discourse::InvalidParameters.new(:user_id) if user.blank?
          
                # keep some user data around to check them against the newly restored database
                @user_info = { id: user.id, username: user.username, email: user.email }
              end
          
              def ensure_we_have_a_filename
                raise BackupRestore::FilenameMissingError if @filename.nil?
              end
          
              def validate_backup_metadata
                @factory.create_meta_data_handler(@filename, @tmp_directory).validate
              end
          
              def reload_site_settings
                log "Reloading site settings..."
                SiteSetting.refresh!
          
                DiscourseEvent.trigger(:site_settings_restored)
          
                if @disable_emails && SiteSetting.disable_emails == "no"
                  log "Disabling outgoing emails for non-staff users..."
                  user = User.find_by_email(@user_info[:email]) || Discourse.system_user
                  SiteSetting.set_and_log(:disable_emails, "non-staff", user)
                end
              end
          
              def run_seed_fu
                log "Running seed fu..."
          
                begin
                  Discourse::Application.load_tasks
                  Rake::Task["db:seed"].invoke
                rescue => ex
                  log "Failed to run seed fu", ex
                end
              end
          
              def clear_category_cache
                log "Clearing category cache..."
                Category.reset_topic_ids_cache
                Category.clear_subcategory_ids
              end
          
              def clear_emoji_cache
                log "Clearing emoji cache..."
                Emoji.clear_cache
              rescue => ex
                log "Something went wrong while clearing emoji cache.", ex
              end
          
              def reload_translations
                log "Reloading translations..."
                TranslationOverride.reload_all_overrides!
              end
          
              def restore_uploads
                if @interactive
                  puts ""
                  puts "Attention! Pausing restore before uploads.".red.bold
                  puts "You can work on the restored database in a separate Rails console."
                  puts ""
                  puts "Press any key to continue with the restore.".bold
                  puts ""
                  STDIN.getch
                end
          
                @uploads_restorer.restore(@tmp_directory)
              end
          
              def notify_user
                return if @success && @user_id == Discourse::SYSTEM_USER_ID
          
                if user = User.find_by_email(@user_info[:email])
                  log "Notifying '#{user.username}' of the end of the restore..."
                  status = @success ? :restore_succeeded : :restore_failed
          
                  logs = Discourse::Utils.logs_markdown(@logger.logs, user: user)
                  post = SystemMessage.create_from_system_user(user, status, logs: logs)
                else
                  log "Could not send notification to '#{@user_info[:username]}' " \
                        "(#{@user_info[:email]}), because the user does not exist."
                end
              rescue => ex
                log "Something went wrong while notifying user.", ex
              end
          
              def clean_up
                log "Cleaning stuff up..."
                @database_restorer.clean_up
                @backup_file_handler.clean_up
                @system.unpause_sidekiq
                @system.disable_readonly_mode if Discourse.readonly_mode?
                @system.mark_restore_as_not_running
              end
          
              def clear_theme_cache
                log "Clear theme cache"
                ThemeField.force_recompilation!
                Theme.expire_site_cache!
                Stylesheet::Manager.cache.clear
              rescue => ex
                log "Something went wrong while clearing theme cache.", ex
              end
          
              def clear_stats
                Discourse.stats.remove("missing_s3_uploads")
              end
          
              def after_restore_hook
                log "Executing the after_restore_hook..."
                DiscourseEvent.trigger(:restore_complete)
              end
            end
          end

This file has been truncated. show original

However, this renames ALL upload filenames, not just the ones that were extracted from the archive. If the database name occurs in an S3 path (which obviously does not change), it changes it as well, causing all images to fail loading.

codinghorror · 2019 年 5 月 29 日午前 12:44

What do you think here @tgxworld?

supermathie · 2019 年 5 月 29 日午後 4:15

Would changing this to include the trailing slash have helped?

DbHelper.remap("uploads/#{previous_db_name}/", "uploads/#{current_db_name}/")

That feels more correct in any case.

RGJ · 2019 年 5 月 29 日午後 4:59

No - I’m talking about filenames like //discourse-cloud-file-uploads.s3.dualstack.us-west-2.amazonaws.com/standard11/uploads/DBNAME/original/1X/abcd.....jpg

So I think we’d need to exclude uploads starting with // or something like that.

tgxworld · 2019 年 5 月 30 日午前 1:34

We’re experiencing the pain of this quite abit while migrating local uploads to S3. Unfortunately, no amount of hacky remaps can fix this properly. What we’re currently working on at the moment is to remove/disallow upload links in Post#raw. The short upload url, ![test](upload://asdikajsdiasds.png), will be used as the public interface for access to uploads in raw.

Once that is done, we will only have to remap the urls in the uploads table and do a rebake of affected posts.

RGJ · 2019 年 5 月 30 日午前 7:30

“only” have to do a rebake… yes, on one hand I think it’s good to get rid of the hacky remaps, on the other hand it seems like the backup/restore and migrate from/to S3 processes are getting more and more complicated and hard.

See also a number of issues in Migrate_from_s3 problems which are kind of related but they’re not all covered by your proposed solution.

sam · 2019 年 5 月 30 日午前 7:52

I do not think so, the moves will get significantly simpler. Walk the uploads table and upload all files to new s3 bucket, rebake and you are done.

They do though become more expensive computation wise cause a rebake is required.

This is a cost I think is worth bearing given we get a correct move at the end of the process. But, we have the flexibility of still doing database replace as we do today for the cooked column and post revisions if that is how we roll, the key change in the process is that we leave posts.raw alone and then have 0% of corruption there. posts.raw and posts_uploads auto generated map will always be stable.

RGJ · 2019 年 5 月 30 日午前 8:00

Ok - good point.

I think a significant improvement would be to walk the uploads table instead of posts (see the topic I linked above) and that would indeed leave the flexibility to replace in posts.cooked

codinghorror · 2019 年 11 月 21 日午後 4:20

@sam、これはまだ問題になっていますか？

sam · 2019 年 11 月 21 日午後 11:02

upload:// リンクを生の Markdown に含めると、これが非常に役立ちます。@gerhard が、ポストの復元時に完全なリベイクをキューに追加するように修正したかどうかはわかりません（そうしていると思います）。

gerhard · 2019 年 11 月 21 日午後 11:40

状況によります。バックアップに最適化された画像が含まれていない場合（これがデフォルト設定です）、アップロードを含むすべての投稿を再焼き付けします。それ以外の場合は、リマップのみを頼ります。

sam · 2019 年 11 月 21 日午後 11:43

安全のために、無条件で再ビルドするように変更すべきでしょう。

gerhard · 2019 年 11 月 21 日午後 11:45

もちろん、変更しました。バックアップとリストアの大きなリファクタリングに組み込まれます。

トピック		返信	表示
False positives on "posts are not remapped to new S3 upload URL" Bug s3	5	159	2025 年 6 月 22 日
Image uploads path in posts will not change after rebake and remap Support	4	1295	2024 年 4 月 27 日
Migrate_to_S3 Fails on Rebake Self-hosting	17	2505	2022 年 9 月 10 日
Bizarre Problems with migrate_from_s3 Development	4	1163	2018 年 12 月 26 日
S3 migration issues- some images got pointed to S3, but with the wrong URL format Support	7	1559	2018 年 12 月 29 日

アップロードのリマッピングが少し熱心すぎる

関連トピック