Refatorei bastante o código. Isso parece estar funcionando bem até o momento. Os campos personalizados que você pode ter não serão importados, mas eu reescrevi essa parte novamente (veja abaixo) e vou limpar minha importação e recomeçar.
Confirmado funcionando do meu Xenforo 2.3 para um Discourse auto-hospedado:
Usuários
Campos de usuário personalizados
Mensagens privadas / Conversas
Galeria de Mídia XenForo
Gerenciador de Recursos Xenforo
Curtidas e reações
Tratamento de e-mails duplicados e inválidos
Tratamento de BB code XF2.3
Galeria de Mídia XF
Recursos XF
Testei isso em um fórum com 750 mil posts, 20 mil membros e 20,7 GB de anexos. A importação leva cerca de 20 horas. Algumas horas adicionais após a importação para o sidekiq se recuperar.
Pontos a observar.
Eu tive que permitir que meu docker acessasse o banco de dados local no CSF.
Os mounts são importantes:
ATTACHMENT_DIR = ENV["XF_ATTACHMENT_DIR"] || "/shared/import/internal_data/attachments"
AVATAR_DIR = ENV["XF_AVATAR_DIR"] || "/shared/import/data/avatars"
Isso evita que você precise mover os dados para a instância do docker. Você está fazendo um link aqui
# frozen_string_literal: true
# Script de Importação do XenForo 2.3 para Discourse
# Configurado especificamente para TurboRenault (dev.turborenault.co.uk)
# Suporta:
# - 750.000+ Posts com Paginação Keyset (Loteamento rápido O(1))
# - 20,7GB de Anexos & Avatares (Resolução de caminho XF 2.3)
# - Campos de Perfil de Usuário Personalizados & Campos Personalizados de Tópico/Post
# - Grupos de Usuários & Membros de Grupos Secundários (Truncamento seguro de colisão de 20 caracteres)
# - Mensagens Privadas / Conversas & Múltiplos destinatários (Com Fallback para Usuário Excluído)
# - Galeria de Mídia XenForo (XFMG)
# - Gerenciador de Recursos XenForo (XFRM com resolução precisa de tópico e atualização)
# - Curtidas & Reações
# - Tratamento Automático de Fallback para E-mails Duplicados & Inválidos
# - Parser de Formatação / BBCode XF 2.3
#
# Execução:
# cd /var/discourse
# ./launcher enter turborenault
# su - discourse
# cd /var/www/discourse
# RAILS_ENV=production bundle exec ruby /shared/xenforo23.rb
require "mysql2"
require "set"
# Resolução robusta de caminho para base.rb
base_path = File.expand_path("../base.rb", __FILE__)
base_path = "/var/www/discourse/script/import_scripts/base.rb" unless File.exist?(base_path)
require base_path
class ImportScripts::XenForo23 < ImportScripts::Base
# Credenciais do Banco de Dados Pré-preenchidas
XENFORO_DB = ENV["XF_DB_NAME"] || "YourDB"
DB_HOST = ENV["XF_DB_HOST"] || "172.17.0.1"
DB_PORT = (ENV["XF_DB_PORT"] || 3306).to_i
DB_USER = ENV["XF_DB_USER"] || "YourUserName"
DB_PASS = ENV["XF_DB_PASS"] || "YourPassword"
TABLE_PREFIX = ENV["XF_TABLE_PREFIX"] || "xf_"
BATCH_SIZE = (ENV["XF_BATCH_SIZE"] || 2000).to_i
# Caminhos Montados
ATTACHMENT_DIR = ENV["XF_ATTACHMENT_DIR"] || "/shared/import/internal_data/attachments"
AVATAR_DIR = ENV["XF_AVATAR_DIR"] || "/shared/import/data/avatars"
# Alternativas de Funcionalidade
IMPORT_PM = ENV.fetch("IMPORT_PM", "true") == "true"
IMPORT_LIKES = ENV.fetch("IMPORT_LIKES", "true") == "true"
IMPORT_XFMG = ENV.fetch("IMPORT_XFMG", "true") == "true"
IMPORT_XFRM = ENV.fetch("IMPORT_XFRM", "true") == "true"
IMPORT_CUSTOM_FIELDS = ENV.fetch("IMPORT_CUSTOM_FIELDS", "true") == "true"
def initialize
super
puts "--> Conectando ao Banco de Dados MySQL '#{XENFORO_DB}' em #{DB_HOST}:#{DB_PORT} como '#{DB_USER}'..."
@client = Mysql2::Client.new(
host: DB_HOST,
port: DB_PORT,
username: DB_USER,
password: DB_PASS,
database: XENFORO_DB,
symbolize_keys: true,
encoding: "utf8mb4",
reconnect: true
)
@category_mappings = {}
@group_mappings = {}
@user_custom_fields_map = {}
@seen_emails = Set.new
@missing_files_count = 0
end
def execute
optimize_discourse_for_import
import_custom_user_field_definitions if IMPORT_CUSTOM_FIELDS
import_users
import_groups
import_categories
import_posts
import_xfrm if IMPORT_XFRM
import_xfmg if IMPORT_XFMG
import_private_messages if IMPORT_PM
import_likes if IMPORT_LIKES
restore_discourse_settings
report_missing_files_summary
end
# =========================================================================
# Ajuste de Desempenho para Discourse Moderno
# =========================================================================
def optimize_discourse_for_import
puts "", "--> Aplicando otimizações de importação de alta velocidade no Discourse..."
begin
SiteSetting.disable_emails = "all" if SiteSetting.respond_to?(:disable_emails=)
SiteSetting.process_with_fastimage = false if SiteSetting.respond_to?(:process_with_fastimage=)
SiteSetting.max_topic_title_length = 500 if SiteSetting.respond_to?(:max_topic_title_length=)
SiteSetting.min_topic_title_length = 1 if SiteSetting.respond_to?(:min_topic_title_length=)
SiteSetting.min_post_length = 1 if SiteSetting.respond_to?(:min_post_length=)
SiteSetting.allow_duplicate_topic_titles = true if SiteSetting.respond_to?(:allow_duplicate_topic_titles=)
RateLimiter.disable rescue nil
rescue StandardError => e
puts " Nota de otimização: #{e.message}"
end
end
def report_missing_files_summary
if @missing_files_count > 0
puts "", " *** #{@missing_files_count} arquivos NÃO foram encontrados no disco durante a importação ***"
else
puts " Todos os arquivos de anexo/avatar encontrados com sucesso."
end
end
def log_missing_file(type, context = {})
@missing_files_count += 1
label = type.to_s.upcase
details = context.map { |k, v| " #{k}: #{v}" }.join("\n")
puts "\n [ARQUIVO FALTANDO: #{label}]\n#{details}"
end
def restore_discourse_settings
puts "", "--> Restaurando configurações padrão do site Discourse..."
begin
SiteSetting.disable_emails = "non_staff" if SiteSetting.respond_to?(:disable_emails=)
RateLimiter.enable rescue nil
rescue StandardError
# ignorar
end
puts "--> Importação Concluída com Sucesso!"
end
# =========================================================================
# 1. Campos de Perfil de Usuário Personalizados
# =========================================================================
def import_custom_user_field_definitions
puts "", "--> Importando Definições de Campos de Usuário Personalizados..."
fields = mysql_query("SELECT field_id, field_type FROM #{TABLE_PREFIX}user_field").to_a
phrases = {}
begin
phrase_rows = mysql_query("SELECT title, phrase_text FROM #{TABLE_PREFIX}phrase WHERE title LIKE 'user_field_title.%' AND language_id = 1").to_a
phrase_rows = mysql_query("SELECT title, phrase_text FROM #{TABLE_PREFIX}phrase WHERE title LIKE 'user_field_title.%'") if phrase_rows.empty?
phrase_rows.each do |p|
fid = p[:title].to_s.sub("user_field_title.", "")
phrases[fid] = p[:phrase_text]
end
rescue StandardError => e
puts " Nota de frase: #{e.message}"
end
fields.each do |row|
field_id = row[:field_id]
title = phrases[field_id].presence || field_id.to_s.tr("_", " ").split.map(&:capitalize).join(" ")
desc = "Campo de perfil personalizado: #{title}"
field_type = case row[:field_type]
when "textbox", "textarea" then "text"
when "select", "radio" then "dropdown"
when "checkbox" then "confirm"
else "text"
end
uf = UserField.find_or_create_by!(name: title) do |f|
f.description = desc
f.field_type = field_type
f.editable = true
f.show_on_profile = true
f.show_on_user_card = true
end
@user_custom_fields_map[field_id] = uf.id
end
puts " Mapeadas #{@user_custom_fields_map.size} definições de campos de perfil personalizados."
end
def fetch_user_custom_fields(xf_user_id)
return {} unless IMPORT_CUSTOM_FIELDS && @user_custom_fields_map.present?
sql = "SELECT field_id, field_value FROM #{TABLE_PREFIX}user_field_value WHERE user_id = #{xf_user_id.to_i}"
rows = mysql_query(sql).to_a
return {} if rows.empty?
custom_fields = {}
rows.each do |row|
discourse_field_id = @user_custom_fields_map[row[:field_id]]
next unless discourse_field_id
val = row[:field_value]
if val.to_s.start_with?("a:", "s:", "i:", "b:")
begin
val = PHP.unserialize(val)
val = val.values.join(", ") if val.is_a?(Hash)
val = val.join(", ") if val.is_a?(Array)
rescue StandardError
# deixar como está
end
elsif val.to_s.start_with?("[", "{")
begin
parsed = JSON.parse(val)
val = parsed.is_a?(Array) ? parsed.join(", ") : val
rescue JSON::ParserError
# deixar como está
end
end
custom_fields["user_field_#{discourse_field_id}"] = val.to_s if val.present?
end
custom_fields
end
# =========================================================================
# 2. Usuários & Avatares
# =========================================================================
def import_users
puts "", "--> Importando Usuários..."
total_count = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}user WHERE user_state = 'valid' AND is_banned = 0").to_a.first[:count]
puts " Encontrados #{total_count} usuários válidos."
last_user_id = 0
processed = 0
loop do
sql = "SELECT u.user_id, u.username, u.email, u.custom_title, u.register_date,
u.last_activity, u.user_group_id, u.is_moderator, u.is_admin, u.is_staff, u.avatar_date
FROM #{TABLE_PREFIX}user u
WHERE u.user_state = 'valid' AND u.is_banned = 0 AND u.user_id > #{last_user_id}
ORDER BY u.user_id ASC
LIMIT #{BATCH_SIZE}"
results = mysql_query(sql).to_a
break if results.empty?
last_user_id = results.last[:user_id]
next if all_records_exist?(:users, results.map { |u| u[:user_id] })
create_users(results, total: total_count, offset: processed) do |user|
username = clean_username(user[:username])
next if username.blank?
email = clean_email(user[:email], user[:user_id])
c_fields = fetch_user_custom_fields(user[:user_id])
{
id: user[:user_id],
email: email,
username: username,
title: user[:custom_title],
created_at: Time.zone.at(user[:register_date]),
last_seen_at: Time.zone.at(user[:last_activity]),
moderator: user[:is_moderator] == 1 || user[:is_staff] == 1,
admin: user[:is_admin] == 1,
custom_fields: c_fields,
post_create_action: proc { |u| import_avatar(user[:user_id], user[:avatar_date], u) }
}
end
processed += results.size
end
end
def clean_username(name)
return "" if name.blank?
name = name.tr(" ", "_").gsub(/[^a-zA-Z0-9_\-\.]/, "")
name.first(60)
end
def clean_email(email, user_id)
email_str = email.to_s.strip.downcase
# Verificar validade padrão
unless email_str.present? && email_str.include?("@") && email_str.match?(/\A[^@\s]+@[^@\s]+\.[^@\s]+\z/)
return "user_#{user_id}@imported.invalid"
end
# Verificar duplicatas em todo o conjunto de dados ou na tabela existente UserEmail do Discourse
if @seen_emails.include?(email_str) || UserEmail.exists?(email: email_str)
prefix = email_str.split("@").first.gsub(/[^a-zA-Z0-9]/, "")
return "duplicate_#{user_id}_#{prefix}@imported.invalid"
end
@seen_emails.add(email_str)
email_str
end
def import_avatar(xf_user_id, avatar_date, imported_user)
return if avatar_date.to_i == 0
group_id = xf_user_id / 1000
filename = File.join(AVATAR_DIR, "l", group_id.to_s, "#{xf_user_id}.jpg")
filename = File.join(AVATAR_DIR, "o", group_id.to_s, "#{xf_user_id}.jpg") unless File.exist?(filename)
unless File.exist?(filename)
log_missing_file(:avatar,
user_id: xf_user_id,
expected_l: File.join(AVATAR_DIR, "l", group_id.to_s, "#{xf_user_id}.jpg"),
expected_o: File.join(AVATAR_DIR, "o", group_id.to_s, "#{xf_user_id}.jpg")
)
return
end
upload = create_upload(imported_user.id, filename, "avatar_#{xf_user_id}.jpg")
return if !upload&.persisted?
imported_user.create_user_avatar if imported_user.user_avatar.nil?
imported_user.user_avatar.update(custom_upload_id: upload.id)
imported_user.update(uploaded_avatar_id: upload.id)
rescue StandardError => e
STDERR.puts "Erro ao importar avatar para o usuário #{xf_user_id}: #{e.message}"
end
# =========================================================================
# 3. Grupos de Usuários & Membros (Truncamento Seguro de Colisão)
# =========================================================================
def import_groups
puts "", "--> Importando Grupos de Usuários e Membros..."
existing_group_names = Set.new(Group.pluck(:name))
xf_groups = mysql_query("SELECT user_group_id, title FROM #{TABLE_PREFIX}user_group").to_a
xf_groups.each do |g|
xf_gid = g[:user_group_id]
raw_title = CGI.unescapeHTML(g[:title].to_s.strip)
# Pular grupos de sistema padrão do XF (1 = Não registrado, 2 = Registrado)
next if [1, 2].include?(xf_gid)
sanitised_name = generate_unique_group_name(raw_title, xf_gid, existing_group_names)
next if sanitised_name.blank?
group = Group.find_by(name: sanitised_name) || Group.create!(
name: sanitised_name,
full_name: raw_title,
visibility_level: Group.visibility_levels[:members]
)
@group_mappings[xf_gid] = group.id
end
puts " Mapeados #{@group_mappings.size} grupos de usuários personalizados."
last_user_id = 0
loop do
relations = mysql_query(
"SELECT user_id, user_group_id
FROM #{TABLE_PREFIX}user_group_relation
WHERE user_id > #{last_user_id}
ORDER BY user_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if relations.empty?
last_user_id = relations.last[:user_id]
relations.each do |r|
discourse_group_id = @group_mappings[r[:user_group_id]]
next unless discourse_group_id
discourse_user_id = user_id_from_imported_user_id(r[:user_id])
next unless discourse_user_id
GroupUser.find_or_create_by!(
group_id: discourse_group_id,
user_id: discourse_user_id
)
end
end
puts " Membros dos grupos importados com sucesso."
end
def generate_unique_group_name(title, group_id, existing_names)
base_slug = title.to_s.parameterize(separator: '_')
base_slug = "group_#{group_id}" if base_slug.blank?
candidate = base_slug.slice(0, 20).chomp('_')
counter = 1
while existing_names.include?(candidate)
suffix = "_#{counter}"
max_base_len = 20 - suffix.length
trimmed_base = base_slug.slice(0, max_base_len).chomp('_')
candidate = "#{trimmed_base}#{suffix}"
counter += 1
end
existing_names.add(candidate)
candidate
end
# =========================================================================
# 4. Categorias (Hierarquia do Fórum)
# =========================================================================
def import_categories
puts "", "--> Importando Categorias..."
nodes = mysql_query(
"SELECT n.node_id, n.title, n.description, n.parent_node_id, n.node_name, n.display_order, n.node_type_id
FROM #{TABLE_PREFIX}node n
WHERE n.node_type_id IN ('Category', 'Forum')
ORDER BY n.parent_node_id ASC, n.display_order ASC"
).to_a
top_level = nodes.select { |n| n[:parent_node_id] == 0 }
create_categories(top_level) do |c|
{
id: c[:node_id],
name: CGI.unescapeHTML(c[:title]),
description: c[:description],
position: c[:display_order],
post_create_action: proc do |cat|
Permalink.find_or_create_by(url: "forums/#{c[:node_name]}.#{c[:node_id]}", category_id: cat.id) if c[:node_name].present?
end
}
end
top_ids = Set.new(top_level.map { |c| c[:node_id] })
second_level = nodes.select { |n| top_ids.include?(n[:parent_node_id]) }
create_categories(second_level) do |c|
{
id: c[:node_id],
name: CGI.unescapeHTML(c[:title]),
description: c[:description],
position: c[:display_order],
parent_category_id: category_id_from_imported_category_id(c[:parent_node_id]),
post_create_action: proc do |cat|
Permalink.find_or_create_by(url: "forums/#{c[:node_name]}.#{c[:node_id]}", category_id: cat.id) if c[:node_name].present?
end
}
end
second_ids = Set.new(second_level.map { |c| c[:node_id] })
deeper = nodes.reject { |n| n[:parent_node_id] == 0 || top_ids.include?(n[:node_id]) || second_ids.include?(n[:node_id]) }
deeper.each do |c|
parent = c
while parent && !second_ids.include?(parent[:node_id]) && !top_ids.include?(parent[:node_id])
parent = nodes.find { |n| n[:node_id] == parent[:parent_node_id] }
end
if parent
@category_mappings[c[:node_id]] = category_id_from_imported_category_id(parent[:node_id])
end
end
end
# =========================================================================
# 5. Tópicos & Posts (Contador ao Vivo para 750.000+ Posts)
# =========================================================================
def import_posts
puts "", "--> Importando Tópicos e Posts (750k Paginação Keyset)..."
total_posts = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}post p INNER JOIN #{TABLE_PREFIX}thread t ON p.thread_id = t.thread_id WHERE p.message_state = 'visible' AND t.discussion_state = 'visible'").to_a.first[:count]
puts " Total de posts a importar: #{total_posts}"
last_post_id = 0
processed = 0
loop do
results = mysql_query(
"SELECT p.post_id AS id,
p.thread_id AS topic_id,
t.node_id AS category_id,
t.title AS title,
t.first_post_id AS first_post_id,
t.view_count AS view_count,
t.prefix_id AS prefix_id,
p.user_id AS user_id,
p.message AS raw,
p.post_date AS created_at
FROM #{TABLE_PREFIX}post p
INNER JOIN #{TABLE_PREFIX}thread t ON p.thread_id = t.thread_id
WHERE p.message_state = 'visible'
AND t.discussion_state = 'visible'
AND p.post_id > #{last_post_id}
ORDER BY p.post_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if results.empty?
last_post_id = results.last[:id]
next if all_records_exist?(:posts, results.map { |r| r[:id] })
create_posts(results, total: total_posts, offset: processed) do |m|
skip = false
mapped = {}
mapped[:id] = m[:id]
mapped[:user_id] = user_id_from_imported_user_id(m[:user_id]) || Discourse::SYSTEM_USER_ID
mapped[:raw] = process_xenforo_post(m[:raw], m[:id])
mapped[:created_at] = Time.zone.at(m[:created_at])
if m[:id] == m[:first_post_id]
cat_id = category_id_from_imported_category_id(m[:category_id]) || @category_mappings[m[:category_id]] || SiteSetting.uncategorized_category_id
mapped[:category] = cat_id
mapped[:title] = CGI.unescapeHTML(m[:title])
mapped[:views] = m[:view_count]
tags = fetch_thread_tags(m[:topic_id])
mapped[:tags] = tags if tags.present?
mapped[:post_create_action] = proc do |pp|
Permalink.find_or_create_by(url: "threads/#{m[:topic_id]}", topic_id: pp.topic_id)
end
else
parent = topic_lookup_from_imported_post_id(m[:first_post_id])
if parent
mapped[:topic_id] = parent[:topic_id]
else
skip = true
end
end
skip ? nil : mapped
end
processed += results.size
end
end
def fetch_thread_tags(thread_id)
sql = "SELECT t.tag FROM #{TABLE_PREFIX}tag_content tc INNER JOIN #{TABLE_PREFIX}tag t ON tc.tag_id = t.tag_id WHERE tc.content_type = 'thread' AND tc.content_id = #{thread_id.to_i}"
rows = mysql_query(sql).to_a
rows.map { |r| DiscourseTagging.clean_tag(r[:tag]) }.compact
end
# =========================================================================
# 6. Gerenciador de Recursos XenForo (XFRM)
# =========================================================================
# =========================================================================
# 6. Gerenciador de Recursos XenForo (XFRM)
# =========================================================================
def import_xfrm
puts "", "--> Importando Gerenciador de Recursos XenForo (XFRM)..."
has_xfrm = mysql_query("SHOW TABLES LIKE '#{TABLE_PREFIX}rm_resource'").to_a.present?
unless has_xfrm
puts " Tabelas XFRM não encontradas, pulando."
return
end
xfrm_cat = Category.find_by_name("Resources") || Category.create!(name: "Resources", user_id: Discourse::SYSTEM_USER_ID)
resources = mysql_query(
"SELECT r.resource_id, r.title, r.tag_line, r.user_id, r.resource_category_id, r.resource_date, r.discussion_thread_id,
t.first_post_id, u.message AS update_message, p.message AS post_message
FROM #{TABLE_PREFIX}rm_resource r
LEFT JOIN #{TABLE_PREFIX}thread t ON t.thread_id = r.discussion_thread_id
LEFT JOIN #{TABLE_PREFIX}post p ON p.post_id = t.first_post_id
LEFT JOIN #{TABLE_PREFIX}rm_resource_update u ON u.resource_update_id = r.description_update_id
WHERE r.resource_state = 'visible'"
).to_a
puts " Encontrados #{resources.size} recursos XFRM."
resources.each do |res|
raw_msg = res[:update_message].presence || res[:post_message].presence
body_text = raw_msg.present? ? process_xenforo_post(raw_msg, res[:first_post_id].to_i) : ""
tagline = res[:tag_line].present? ? CGI.unescapeHTML(res[:tag_line]) : ""
raw = ""
raw += "**#{tagline}**\n\n" if tagline.present?
raw += body_text.present? ? body_text : "*(Nenhuma descrição fornecida)*"
# --- NOVO: BUSCAR ARQUIVOS DE RECURSO REAIS (ZIPS, PDFS) ---
# Atualizado 'v.version_id' para 'v.resource_version_id' do XenForo 2
version_files_sql = "
SELECT a.attachment_id, a.data_id, d.filename, d.file_hash, d.user_id
FROM #{TABLE_PREFIX}attachment a
INNER JOIN #{TABLE_PREFIX}attachment_data d ON a.data_id = d.data_id
INNER JOIN #{TABLE_PREFIX}rm_resource_version v ON v.resource_version_id = a.content_id
WHERE a.content_type = 'resource_version' AND v.resource_id = #{res[:resource_id]}
"
resource_files = mysql_query(version_files_sql).to_a
if resource_files.any?
raw += "\n\n### Arquivos para Download\n"
resource_files.each do |file|
upload = import_xf_attachment_file(file[:data_id], file[:file_hash], file[:user_id], file[:filename])
if upload&.persisted?
html = @uploader.html_for_upload(upload, file[:filename])
raw += "\n* #{html}"
end
end
end
version_urls_sql = "
SELECT version_string, download_url
FROM #{TABLE_PREFIX}rm_resource_version
WHERE resource_id = #{res[:resource_id]} AND download_url != ''
"
external_urls = mysql_query(version_urls_sql).to_a
if external_urls.any?
raw += "\n\n### Downloads Externos\n"
external_urls.each do |ext|
raw += "\n* [Versão #{ext[:version_string]}](#{ext[:download_url]})"
end
end
# --- FIM DA BUSCA DE ARQUIVOS NOVA ---
if res[:discussion_thread_id].to_i > 0 && res[:first_post_id].to_i > 0
topic_map = topic_lookup_from_imported_post_id(res[:first_post_id].to_i)
if topic_map
raw += "\n\n---\n*Tópico de Discussão Original: [Ver Aqui](/t/-/#{topic_map[:topic_id]})*"
end
end
user_id = user_id_from_imported_user_id(res[:user_id]) || Discourse::SYSTEM_USER_ID
create_post = {
id: "xfrm_v5_#{res[:resource_id]}", # Atualizado para v5 para limpar o cache corrompido
user_id: user_id,
category: xfrm_cat.id,
title: CGI.unescapeHTML(res[:title]),
raw: raw,
created_at: Time.zone.at(res[:resource_date])
}
create_posts([create_post], total: resources.size, offset: 0) { |p| p }
end
end
# =========================================================================
# 7. Galeria de Mídia XenForo (XFMG)
# =========================================================================
def import_xfmg
puts "", "--> Importando Galeria de Mídia XenForo (XFMG)..."
has_xfmg = mysql_query("SHOW TABLES LIKE '#{TABLE_PREFIX}mg_media_item'").to_a.present?
unless has_xfmg
puts " Tabelas XFMG não encontradas, pulando."
return
end
xfmg_cat = Category.find_by_name("Media Gallery") || Category.create!(name: "Media Gallery", user_id: Discourse::SYSTEM_USER_ID)
items = mysql_query(
"SELECT m.media_id, m.title, m.description, m.media_type, m.media_tag, m.user_id, m.media_date
FROM #{TABLE_PREFIX}mg_media_item m
WHERE m.media_state = 'visible'"
).to_a
puts " Encontrados #{items.size} itens de mídia XFMG."
items.each do |item|
raw = item[:description].presence || ""
# Se for um vídeo incorporado (YouTube, Vimeo, etc.), incluir a URL do embed
if item[:media_type] == "embed" && item[:media_tag].present?
raw = "#{item[:media_tag]}\n\n#{raw}"
else
# Processar anexos de imagem usando o 'xfmg_media' content_type do XF 2.x
raw = process_xf_attachments("xfmg_media", raw, item[:media_id])
end
raw = "*(Nenhum conteúdo de mídia)*" if raw.blank?
user_id = user_id_from_imported_user_id(item[:user_id]) || Discourse::SYSTEM_USER_ID
post_data = {
id: "xfmg_#{item[:media_id]}",
user_id: user_id,
category: xfmg_cat.id,
title: CGI.unescapeHTML(item[:title].presence || "Media #{item[:media_id]}"),
raw: raw,
created_at: Time.zone.at(item[:media_date])
}
create_posts([post_data], total: items.size, offset: 0) { |p| p }
end
end
# =========================================================================
# 8. Mensagens Privadas (Conversas)
# =========================================================================
def import_private_messages
puts "", "--> Importando Mensagens Privadas (Conversas)..."
last_conv_id = 0
total_count = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}conversation_master").to_a.first[:count]
puts " Encontradas #{total_count} conversas de mensagens privadas."
processed = 0
loop do
posts = mysql_query(
"SELECT c.conversation_id, c.title, m.message_id, m.message, m.user_id, m.message_date, c.first_message_id
FROM #{TABLE_PREFIX}conversation_master c
INNER JOIN #{TABLE_PREFIX}conversation_message m ON m.conversation_id = c.conversation_id
WHERE c.conversation_id > #{last_conv_id}
ORDER BY c.conversation_id ASC, m.message_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if posts.empty?
last_conv_id = posts.last[:conversation_id]
next if all_records_exist?(:posts, posts.map { |p| "pm_#{p[:message_id]}" })
create_posts(posts, total: total_count, offset: processed) do |post|
user_id = user_id_from_imported_user_id(post[:user_id]) || Discourse::SYSTEM_USER_ID
message_id = "pm_#{post[:message_id]}"
raw = process_xenforo_post(post[:message], 0)
next if raw.blank?
msg = {
id: message_id,
user_id: user_id,
raw: raw,
created_at: Time.zone.at(post[:message_date].to_i),
import_mode: true
}
if post[:message_id] == post[:first_message_id]
msg[:title] = CGI.unescapeHTML(post[:title])
msg[:archetype] = Archetype.private_message
recipients_sql = "SELECT user_id FROM #{TABLE_PREFIX}conversation_recipient WHERE conversation_id = #{post[:conversation_id].to_i}"
recip_rows = mysql_query(recipients_sql).to_a
recip_user_ids = recip_rows.map { |r| user_id_from_imported_user_id(r[:user_id]) }.compact
target_usernames = User.where(id: recip_user_ids).pluck(:username)
if target_usernames.blank?
fallback_user = User.find_by(id: Discourse::SYSTEM_USER_ID) || User.admins.first
target_usernames = [fallback_user.username] if fallback_user
end
msg[:target_usernames] = target_usernames.join(",")
else
first_msg_map = topic_lookup_from_imported_post_id("pm_#{post[:first_message_id]}")
if first_msg_map
msg[:topic_id] = first_msg_map[:topic_id]
else
next
end
end
msg
end
processed += posts.size
if processed % 5000 == 0
ActiveRecord::Base.connection.clear_query_cache
GC.start
end
end
end
# =========================================================================
# 9. Reações & Curtidas
# =========================================================================
def import_likes
puts "", "--> Limpando curtidas poluídas e redefinindo caches de tentativas anteriores..."
# Limpar as ações brutas
PostAction.where(post_action_type_id: PostActionType.types[:like]).delete_all
# Zerar os contadores em cache na interface
Post.update_all(like_count: 0)
UserStat.update_all(likes_given: 0, likes_received: 0)
puts "--> Importando Reações de Posts do XenForo 2.3 (Curtidas)..."
# Contar usando values.first para segurança entre drivers
total_count = mysql_query(
"SELECT COUNT(*) FROM #{TABLE_PREFIX}reaction_content WHERE content_type = 'post' AND is_counted = 1"
).first.values.first
puts " Encontradas #{total_count} reações de post ativas."
last_reaction_content_id = 0
processed = 0
loop do
# Paginar com segurança usando a chave primária (reaction_content_id)
reactions = mysql_query(<<-SQL).to_a
SELECT reaction_content_id, content_id, reaction_user_id, reaction_date
FROM #{TABLE_PREFIX}reaction_content
WHERE content_type = 'post'
AND is_counted = 1
AND reaction_content_id > #{last_reaction_content_id}
ORDER BY reaction_content_id ASC
LIMIT #{BATCH_SIZE}
SQL
break if reactions.empty?
# Extrair com segurança a maior chave primária para o próximo lote de paginação
last_row = reactions.last
last_reaction_content_id = (last_row[:reaction_content_id] || last_row["reaction_content_id"]).to_i
create_likes(reactions, total: total_count, offset: processed) do |row|
xf_post_id = row[:content_id] || row["content_id"]
xf_user_id = row[:reaction_user_id] || row["reaction_user_id"]
xf_date = row[:reaction_date] || row["reaction_date"]
# PASSAR IDS BRUTOS DO XENFORO DIRETAMENTE
# O helper `create_likes` do Discourse traduz esses automaticamente em segundo plano.
{
post_id: xf_post_id,
user_id: xf_user_id,
created_at: Time.zone.at(xf_date.to_i)
}
end
processed += reactions.size
if processed % 10000 == 0
ActiveRecord::Base.connection.clear_query_cache
GC.start
end
end
end
# =========================================================================
# 10. Parser BBCode & Transformador de Formatação
# =========================================================================
def process_xenforo_post(raw, import_id)
return "" if raw.blank?
s = raw.dup
s = s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "") unless s.valid_encoding?
s.gsub!(%r{<!-- s(\S+) --><img (?:[^>]+) /><!-- s(?:\S+) -->}, '\1')
s.gsub!(%r{<!-- \w --><a(?:.+)href="(\S+)"(?:.*)>(.+)</a><!-- \w -->}, '[\2](\1)')
s.gsub!(/:(?:\w{8})\]/, "]")
s = CGI.unescapeHTML(s)
s.gsub!(/\[QUOTE="?([^,\]]+)(?:,\s*post:\s*(\d+))?(?:,\s*member:\s*\d+)"?\)]) do
username = $1
imported_post_id = $2
if imported_post_id.present?
topic_mapping = topic_lookup_from_imported_post_id(imported_post_id.to_i)
if topic_mapping
"\n[quote=\"#{username}, post:#{topic_mapping[:post_number]}, topic:#{topic_mapping[:topic_id]}\"]\n"
else
"\n[quote=\"#{username}\"]\n"
end
else
"\n[quote=\"#{username}\"]\n"
end
end
s.gsub!(%r{\[/QUOTE\]}i, "\n[/quote]\n")
s.gsub!(%r{\[HEADING=1\](.+?)\[/HEADING\]}i) { "\n# #{$1}\n" }
s.gsub!(%r{\[HEADING=2\](.+?)\[/HEADING\]}i) { "\n## #{$1}\n" }
s.gsub!(%r{\[HEADING=3\](.+?)\[/HEADING\]}i) { "\n### #{$1}\n" }
s.gsub!(%r{\[SPOILER="?([^\]]*?)"?\](.*?)\[/SPOILER\]}im) do
title = $1.presence || "Spoiler"
content = $2
"\n[details=\"#{title}\"]\n#{content}\n[/details]\n"
end
s.gsub!(%r{\[CODE="?([a-zA-Z0-9_\-+]*)"?\](.*?)\[/CODE\]}im) do
lang = $1.presence || ""
code = $2
"\n```#{lang}\n#{code}\n```\n"
end
s.gsub!(%r{\[USER=\d+\]@?(.+?)\[/USER\]}i, '@\1')
s.gsub!(%r{\[MEDIA=youtube\](.+?)\[/MEDIA\]}i, 'https://www.youtube.com/watch?v=\1')
s.gsub!(%r{\[MEDIA=[^\]]+\](.+?)\[/MEDIA\]}i, '\1')
# Remove tags BBCode de apresentação que o Markdown não utiliza
s.gsub!(%r{\[/?(?:LEFT|RIGHT|CENTER|JUSTIFY|FONT|SIZE|COLOR|INDENT)(?:=[^\]]+)?\]}i, '')
# Processar e substituir tags [ATTACH] por uploads do Discourse
s = process_xf_attachments("post", s, import_id) if import_id.to_i > 0
s.strip
end
# =========================================================================
# 11. Resolução de Anexos e Tratamento de Arquivos
# =========================================================================
def process_xf_attachments(content_type, text, content_id)
sql = "
SELECT a.attachment_id, a.data_id, d.filename, d.file_hash, d.user_id
FROM #{TABLE_PREFIX}attachment a
INNER JOIN #{TABLE_PREFIX}attachment_data d ON a.data_id = d.data_id
WHERE a.content_type = '#{@client.escape(content_type.to_s)}'
AND a.content_id = #{content_id.to_i}
"
attachments = mysql_query(sql).to_a
return text if attachments.empty?
embedded_ids = Set.new
attachments.each do |att|
att_id = att[:attachment_id]
upload = import_xf_attachment_file(att[:data_id], att[:file_hash], att[:user_id], att[:filename])
next unless upload&.persisted?
html = @uploader.html_for_upload(upload, att[:filename])
# Correspondência com [ATTACH]123[/ATTACH] padrão, bem como variantes de atributos do XF 2.3 como [ATTACH type="full" alt="..."]123[/ATTACH]
pattern = %r{\[ATTACH[^\]]*\]\s*#{att_id}\s*\[/ATTACH\]}i
if text.match?(pattern)
text.gsub!(pattern, "\n#{html}\n")
embedded_ids.add(att_id)
end
end
# Anexar quaisquer anexos que não foram posicionados inline no corpo do texto
unattached = attachments.reject { |a| embedded_ids.include?(a[:attachment_id]) }
if unattached.any?
text += "\n\n"
unattached.each do |att|
upload = import_xf_attachment_file(att[:data_id], att[:file_hash], att[:user_id], att[:filename])
next unless upload&.persisted?
html = @uploader.html_for_upload(upload, att[:filename])
text += "#{html}\n"
end
end
# Passada final de limpeza: remover qualquer marcação [ATTACH] órfã ou ausente
text.gsub!(%r{\[ATTACH[^\]]*\]\s*\d*\s*\[/ATTACH\]}i, '')
text
end
def import_xf_attachment_file(data_id, file_hash, owner_id, original_filename)
group_id = data_id.to_i / 1000
exact_name = "#{data_id}-#{file_hash}.data"
path_grouped = Pathname.new(File.join(ATTACHMENT_DIR, group_id.to_s, exact_name))
path_flat = Pathname.new(File.join(ATTACHMENT_DIR, exact_name))
path = if File.exist?(path_grouped)
path_grouped
elsif File.exist?(path_flat)
path_flat
else
glob_grouped = Dir.glob(File.join(ATTACHMENT_DIR, group_id.to_s, "#{data_id}-*.data")).first
glob_flat = Dir.glob(File.join(ATTACHMENT_DIR, "#{data_id}-*.data")).first
found = glob_grouped || glob_flat
if found
Pathname.new(found)
else
log_missing_file(:attachment,
data_id: data_id,
filename: original_filename,
db_hash: file_hash,
expected_grouped: path_grouped.to_s,
expected_flat: path_flat.to_s
)
return nil
end
end
discourse_user_id = user_id_from_imported_user_id(owner_id) || Discourse::SYSTEM_USER_ID
temp_path = path.dirname.join(original_filename)
FileUtils.cp(path, temp_path)
upload = create_upload(discourse_user_id, temp_path, original_filename)
FileUtils.rm(temp_path) if File.exist?(temp_path)
upload
rescue StandardError => e
STDERR.puts "Falha ao processar anexo data_id #{data_id}: #{e.message}"
nil
end
def mysql_query(sql)
@client.query(sql)
end
end
ImportScripts::XenForo23.new.perform
Alguns usuários falharam na importação devido a endereços de e-mail estranhos criados por spambots. E-mails como Peter.Today.Rain.Import.Something@Dave.Gmail.com, por exemplo.
Vou atualizar/editar meu código de importação acima assim que os campos personalizados estiverem disponíveis, pois eles são importantes para mim.
.
