Ich habe einen Großteil des Codes refaktoriert. Das scheint derzeit recht gut zu funktionieren. Die benutzerdefinierten Felder, die du möglicherweise hast, werden nicht übernommen, aber ich habe das erneut umgeschrieben (siehe unten) und werde meinen Import bereinigen und von vorne beginnen.
Bestätigt funktioniert auf meinem Xenforo 2.3 zu selbst gehostetem Discourse :
Benutzer
Benutzerdefinierte Benutzerfelder
PN / Unterhaltungen
XenForo Media Gallery
Xenforo Resource Manager
Likes und Reaktionen
Duplikat- und ungültige E-Mail-Verarbeitung
XF2.3 BB-Code-Verarbeitung
XFMedia Gallery
XF Resources
Ich habe dies auf 750k Beiträgen, 20k Mitgliedern und 20,7GB Anhängen getestet. Der Import dauert etwa 20 Stunden. Nach dem Import braucht Sidekiq noch ein paar Stunden, um aufzuholen.
Hinweise.
Ich musste es meinem Docker erlauben, in CSF auf die lokale DB zuzugreifen.
Die Mounts sind wichtig :
ATTACHMENT_DIR = ENV["XF_ATTACHMENT_DIR"] || "/shared/import/internal_data/attachments"
AVATAR_DIR = ENV["XF_AVATAR_DIR"] || "/shared/import/data/avatars"
Das spart dir die Mühe, die Daten in die Docker-Instanz zu verschieben. Du verlinkst hier
# frozen_string_literal: true
# XenForo 2.3 zu Discourse Import-Skript
# Speziell für TurboRenault (dev.turborenault.co.uk) konfiguriert
# Unterstützt:
# - 750.000+ Beiträge mit Keyset-Pagination (O(1) schnelles Batching)
# - 20,7GB Anhänge & Avatare (XF 2.3 Pfadauflösung)
# - Benutzerdefinierte Profilfelder & Benutzerdefinierte Themen-/Beitragfelder
# - Benutzergruppen & Sekundäre Gruppenmitgliedschaften (Kollisionsfreie 20-Zeichen-Truncation)
# - Private Nachrichten / Unterhaltungen & Mehrere Empfänger (Mit Fallback für gelöschte Benutzer)
# - XenForo Media Gallery (XFMG)
# - XenForo Resource Manager (XFRM mit genauer Themen- und Update-Auflösung)
# - Likes & Reaktionen
# - Automatische Fallback-Verarbeitung für Duplikate & ungültige E-Mails
# - XF 2.3 BBCode / Formatierungs-Parser
#
# Ausführung:
# cd /var/discourse
# ./launcher enter turborenault
# su - discourse
# cd /var/www/discourse
# RAILS_ENV=production bundle exec ruby /shared/xenforo23.rb
require "mysql2"
require "set"
# Robuste Pfadauflösung für base.rb
base_path = File.expand_path("../base.rb", __FILE__)
base_path = "/var/www/discourse/script/import_scripts/base.rb" unless File.exist?(base_path)
require base_path
class ImportScripts::XenForo23 < ImportScripts::Base
# Vorgefüllte Datenbank-Zugangsdaten
XENFORO_DB = ENV["XF_DB_NAME"] || "YourDB"
DB_HOST = ENV["XF_DB_HOST"] || "172.17.0.1"
DB_PORT = (ENV["XF_DB_PORT"] || 3306).to_i
DB_USER = ENV["XF_DB_USER"] || "YourUserName"
DB_PASS = ENV["XF_DB_PASS"] || "YourPassword"
TABLE_PREFIX = ENV["XF_TABLE_PREFIX"] || "xf_"
BATCH_SIZE = (ENV["XF_BATCH_SIZE"] || 2000).to_i
# Eingebundene Pfade
ATTACHMENT_DIR = ENV["XF_ATTACHMENT_DIR"] || "/shared/import/internal_data/attachments"
AVATAR_DIR = ENV["XF_AVATAR_DIR"] || "/shared/import/data/avatars"
# Funktions-Toggles
IMPORT_PM = ENV.fetch("IMPORT_PM", "true") == "true"
IMPORT_LIKES = ENV.fetch("IMPORT_LIKES", "true") == "true"
IMPORT_XFMG = ENV.fetch("IMPORT_XFMG", "true") == "true"
IMPORT_XFRM = ENV.fetch("IMPORT_XFRM", "true") == "true"
IMPORT_CUSTOM_FIELDS = ENV.fetch("IMPORT_CUSTOM_FIELDS", "true") == "true"
def initialize
super
puts "--> Verbinde mit MySQL-Datenbank '#{XENFORO_DB}' auf #{DB_HOST}:#{DB_PORT} als '#{DB_USER}'..."
@client = Mysql2::Client.new(
host: DB_HOST,
port: DB_PORT,
username: DB_USER,
password: DB_PASS,
database: XENFORO_DB,
symbolize_keys: true,
encoding: "utf8mb4",
reconnect: true
)
@category_mappings = {}
@group_mappings = {}
@user_custom_fields_map = {}
@seen_emails = Set.new
@missing_files_count = 0
end
def execute
optimize_discourse_for_import
import_custom_user_field_definitions if IMPORT_CUSTOM_FIELDS
import_users
import_groups
import_categories
import_posts
import_xfrm if IMPORT_XFRM
import_xfmg if IMPORT_XFMG
import_private_messages if IMPORT_PM
import_likes if IMPORT_LIKES
restore_discourse_settings
report_missing_files_summary
end
# =========================================================================
# Leistungsoptimierung für modernen Discourse
# =========================================================================
def optimize_discourse_for_import
puts "", "--> Wende Hochgeschwindigkeits-Importoptimierungen für Discourse an..."
begin
SiteSetting.disable_emails = "all" if SiteSetting.respond_to?(:disable_emails=)
SiteSetting.process_with_fastimage = false if SiteSetting.respond_to?(:process_with_fastimage=)
SiteSetting.max_topic_title_length = 500 if SiteSetting.respond_to?(:max_topic_title_length=)
SiteSetting.min_topic_title_length = 1 if SiteSetting.respond_to?(:min_topic_title_length=)
SiteSetting.min_post_length = 1 if SiteSetting.respond_to?(:min_post_length=)
SiteSetting.allow_duplicate_topic_titles = true if SiteSetting.respond_to?(:allow_duplicate_topic_titles=)
RateLimiter.disable rescue nil
rescue StandardError => e
puts " Optimierungshinweis: #{e.message}"
end
end
def report_missing_files_summary
if @missing_files_count > 0
puts "", " *** #{@missing_files_count} Dateien wurden während des Imports nicht auf der Festplatte gefunden ***"
else
puts " Alle Anhangs-/Avatar-Dateien erfolgreich gefunden."
end
end
def log_missing_file(type, context = {})
@missing_files_count += 1
label = type.to_s.upcase
details = context.map { |k, v| " #{k}: #{v}" }.join("\n")
puts "\n [FEHLT #{label}]\n#{details}"
end
def restore_discourse_settings
puts "", "--> Stellt Standard-Discourse-Siteeinstellungen wieder her..."
begin
SiteSetting.disable_emails = "non_staff" if SiteSetting.respond_to?(:disable_emails=)
RateLimiter.enable rescue nil
rescue StandardError
# ignorieren
end
puts "--> Import erfolgreich abgeschlossen!"
end
# =========================================================================
# 1. Benutzerdefinierte Profilfelder
# =========================================================================
def import_custom_user_field_definitions
puts "", "--> Importiere Definitionen benutzerdefinierter Benutzerfelder..."
fields = mysql_query("SELECT field_id, field_type FROM #{TABLE_PREFIX}user_field").to_a
phrases = {}
begin
phrase_rows = mysql_query("SELECT title, phrase_text FROM #{TABLE_PREFIX}phrase WHERE title LIKE 'user_field_title.%' AND language_id = 1").to_a
phrase_rows = mysql_query("SELECT title, phrase_text FROM #{TABLE_PREFIX}phrase WHERE title LIKE 'user_field_title.%'") if phrase_rows.empty?
phrase_rows.each do |p|
fid = p[:title].to_s.sub("user_field_title.", "")
phrases[fid] = p[:phrase_text]
end
rescue StandardError => e
puts " Phrasen-Hinweis: #{e.message}"
end
fields.each do |row|
field_id = row[:field_id]
title = phrases[field_id].presence || field_id.to_s.tr("_", " ").split.map(&:capitalize).join(" ")
desc = "Benutzerdefiniertes Profilfeld: #{title}"
field_type = case row[:field_type]
when "textbox", "textarea" then "text"
when "select", "radio" then "dropdown"
when "checkbox" then "confirm"
else "text"
end
uf = UserField.find_or_create_by!(name: title) do |f|
f.description = desc
f.field_type = field_type
f.editable = true
f.show_on_profile = true
f.show_on_user_card = true
end
@user_custom_fields_map[field_id] = uf.id
end
puts " Gemappt #{@user_custom_fields_map.size} Definitionen benutzerdefinierter Profilfelder."
end
def fetch_user_custom_fields(xf_user_id)
return {} unless IMPORT_CUSTOM_FIELDS && @user_custom_fields_map.present?
sql = "SELECT field_id, field_value FROM #{TABLE_PREFIX}user_field_value WHERE user_id = #{xf_user_id.to_i}"
rows = mysql_query(sql).to_a
return {} if rows.empty?
custom_fields = {}
rows.each do |row|
discourse_field_id = @user_custom_fields_map[row[:field_id]]
next unless discourse_field_id
val = row[:field_value]
if val.to_s.start_with?("a:", "s:", "i:", "b:")
begin
val = PHP.unserialize(val)
val = val.values.join(", ") if val.is_a?(Hash)
val = val.join(", ") if val.is_a?(Array)
rescue StandardError
# so belassen
end
elsif val.to_s.start_with?("[", "{")
begin
parsed = JSON.parse(val)
val = parsed.is_a?(Array) ? parsed.join(", ") : val
rescue JSON::ParserError
# so belassen
end
end
custom_fields["user_field_#{discourse_field_id}"] = val.to_s if val.present?
end
custom_fields
end
# =========================================================================
# 2. Benutzer & Avatare
# =========================================================================
def import_users
puts "", "--> Importiere Benutzer..."
total_count = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}user WHERE user_state = 'valid' AND is_banned = 0").to_a.first[:count]
puts " Gefunden #{total_count} gültige Benutzer."
last_user_id = 0
processed = 0
loop do
sql = "SELECT u.user_id, u.username, u.email, u.custom_title, u.register_date,
u.last_activity, u.user_group_id, u.is_moderator, u.is_admin, u.is_staff, u.avatar_date
FROM #{TABLE_PREFIX}user u
WHERE u.user_state = 'valid' AND u.is_banned = 0 AND u.user_id > #{last_user_id}
ORDER BY u.user_id ASC
LIMIT #{BATCH_SIZE}"
results = mysql_query(sql).to_a
break if results.empty?
last_user_id = results.last[:user_id]
next if all_records_exist?(:users, results.map { |u| u[:user_id] })
create_users(results, total: total_count, offset: processed) do |user|
username = clean_username(user[:username])
next if username.blank?
email = clean_email(user[:email], user[:user_id])
c_fields = fetch_user_custom_fields(user[:user_id])
{
id: user[:user_id],
email: email,
username: username,
title: user[:custom_title],
created_at: Time.zone.at(user[:register_date]),
last_seen_at: Time.zone.at(user[:last_activity]),
moderator: user[:is_moderator] == 1 || user[:is_staff] == 1,
admin: user[:is_admin] == 1,
custom_fields: c_fields,
post_create_action: proc { |u| import_avatar(user[:user_id], user[:avatar_date], u) }
}
end
processed += results.size
end
end
def clean_username(name)
return "" if name.blank?
name = name.tr(" ", "_").gsub(/[^a-zA-Z0-9_\-\.]/, "")
name.first(60)
end
def clean_email(email, user_id)
email_str = email.to_s.strip.downcase
# Prüfe auf Standardgültigkeit
unless email_str.present? && email_str.include?("@") && email_str.match?(/\A[^@\s]+@[^@\s]+\.[^@\s]+\z/)
return "user_#{user_id}@imported.invalid"
end
# Prüfe auf Duplikate im Datensatz oder in der bestehenden Discourse UserEmail-Tabelle
if @seen_emails.include?(email_str) || UserEmail.exists?(email: email_str)
prefix = email_str.split("@").first.gsub(/[^a-zA-Z0-9]/, "")
return "duplicate_#{user_id}_#{prefix}@imported.invalid"
end
@seen_emails.add(email_str)
email_str
end
def import_avatar(xf_user_id, avatar_date, imported_user)
return if avatar_date.to_i == 0
group_id = xf_user_id / 1000
filename = File.join(AVATAR_DIR, "l", group_id.to_s, "#{xf_user_id}.jpg")
filename = File.join(AVATAR_DIR, "o", group_id.to_s, "#{xf_user_id}.jpg") unless File.exist?(filename)
unless File.exist?(filename)
log_missing_file(:avatar,
user_id: xf_user_id,
expected_l: File.join(AVATAR_DIR, "l", group_id.to_s, "#{xf_user_id}.jpg"),
expected_o: File.join(AVATAR_DIR, "o", group_id.to_s, "#{xf_user_id}.jpg")
)
return
end
upload = create_upload(imported_user.id, filename, "avatar_#{xf_user_id}.jpg")
return if !upload&.persisted?
imported_user.create_user_avatar if imported_user.user_avatar.nil?
imported_user.user_avatar.update(custom_upload_id: upload.id)
imported_user.update(uploaded_avatar_id: upload.id)
rescue StandardError => e
STDERR.puts "Fehler beim Importieren des Avatars für Benutzer #{xf_user_id}: #{e.message}"
end
# =========================================================================
# 3. Benutzergruppen & Mitgliedschaften (Kollisionsfreie Truncation)
# =========================================================================
def import_groups
puts "", "--> Importiere Benutzergruppen und Mitgliedschaften..."
existing_group_names = Set.new(Group.pluck(:name))
xf_groups = mysql_query("SELECT user_group_id, title FROM #{TABLE_PREFIX}user_group").to_a
xf_groups.each do |g|
xf_gid = g[:user_group_id]
raw_title = CGI.unescapeHTML(g[:title].to_s.strip)
# Überspringe Standard-XF-Systemgruppen (1 = Nicht registriert, 2 = Registriert)
next if [1, 2].include?(xf_gid)
sanitised_name = generate_unique_group_name(raw_title, xf_gid, existing_group_names)
next if sanitised_name.blank?
group = Group.find_by(name: sanitised_name) || Group.create!(
name: sanitised_name,
full_name: raw_title,
visibility_level: Group.visibility_levels[:members]
)
@group_mappings[xf_gid] = group.id
end
puts " Gemappt #{@group_mappings.size} benutzerdefinierte Benutzergruppen."
last_user_id = 0
loop do
relations = mysql_query(
"SELECT user_id, user_group_id
FROM #{TABLE_PREFIX}user_group_relation
WHERE user_id > #{last_user_id}
ORDER BY user_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if relations.empty?
last_user_id = relations.last[:user_id]
relations.each do |r|
discourse_group_id = @group_mappings[r[:user_group_id]]
next unless discourse_group_id
discourse_user_id = user_id_from_imported_user_id(r[:user_id])
next unless discourse_user_id
GroupUser.find_or_create_by!(
group_id: discourse_group_id,
user_id: discourse_user_id
)
end
end
puts " Gruppenmitgliedschaften erfolgreich importiert."
end
def generate_unique_group_name(title, group_id, existing_names)
base_slug = title.to_s.parameterize(separator: '_')
base_slug = "group_#{group_id}" if base_slug.blank?
candidate = base_slug.slice(0, 20).chomp('_')
counter = 1
while existing_names.include?(candidate)
suffix = "_#{counter}"
max_base_len = 20 - suffix.length
trimmed_base = base_slug.slice(0, max_base_len).chomp('_')
candidate = "#{trimmed_base}#{suffix}"
counter += 1
end
existing_names.add(candidate)
candidate
end
# =========================================================================
# 4. Kategorien (Forum-Hierarchie)
# =========================================================================
def import_categories
puts "", "--> Importiere Kategorien..."
nodes = mysql_query(
"SELECT n.node_id, n.title, n.description, n.parent_node_id, n.node_name, n.display_order, n.node_type_id
FROM #{TABLE_PREFIX}node n
WHERE n.node_type_id IN ('Category', 'Forum')
ORDER BY n.parent_node_id ASC, n.display_order ASC"
).to_a
top_level = nodes.select { |n| n[:parent_node_id] == 0 }
create_categories(top_level) do |c|
{
id: c[:node_id],
name: CGI.unescapeHTML(c[:title]),
description: c[:description],
position: c[:display_order],
post_create_action: proc do |cat|
Permalink.find_or_create_by(url: "forums/#{c[:node_name]}.#{c[:node_id]}", category_id: cat.id) if c[:node_name].present?
end
}
end
top_ids = Set.new(top_level.map { |c| c[:node_id] })
second_level = nodes.select { |n| top_ids.include?(n[:parent_node_id]) }
create_categories(second_level) do |c|
{
id: c[:node_id],
name: CGI.unescapeHTML(c[:title]),
description: c[:description],
position: c[:display_order],
parent_category_id: category_id_from_imported_category_id(c[:parent_node_id]),
post_create_action: proc do |cat|
Permalink.find_or_create_by(url: "forums/#{c[:node_name]}.#{c[:node_id]}", category_id: cat.id) if c[:node_name].present?
end
}
end
second_ids = Set.new(second_level.map { |c| c[:node_id] })
deeper = nodes.reject { |n| n[:parent_node_id] == 0 || top_ids.include?(n[:node_id]) || second_ids.include?(n[:node_id]) }
deeper.each do |c|
parent = c
while parent && !second_ids.include?(parent[:node_id]) && !top_ids.include?(parent[:node_id])
parent = nodes.find { |n| n[:node_id] == parent[:parent_node_id] }
end
if parent
@category_mappings[c[:node_id]] = category_id_from_imported_category_id(parent[:node_id])
end
end
end
# =========================================================================
# 5. Themen & Beiträge (Live-Zähler für 750.000+ Beiträge)
# =========================================================================
def import_posts
puts "", "--> Importiere Themen und Beiträge (750k Keyset-Pagination)..."
total_posts = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}post p INNER JOIN #{TABLE_PREFIX}thread t ON p.thread_id = t.thread_id WHERE p.message_state = 'visible' AND t.discussion_state = 'visible'").to_a.first[:count]
puts " Gesamtzahl der zu importierenden Beiträge: #{total_posts}"
last_post_id = 0
processed = 0
loop do
results = mysql_query(
"SELECT p.post_id AS id,
p.thread_id AS topic_id,
t.node_id AS category_id,
t.title AS title,
t.first_post_id AS first_post_id,
t.view_count AS view_count,
t.prefix_id AS prefix_id,
p.user_id AS user_id,
p.message AS raw,
p.post_date AS created_at
FROM #{TABLE_PREFIX}post p
INNER JOIN #{TABLE_PREFIX}thread t ON p.thread_id = t.thread_id
WHERE p.message_state = 'visible'
AND t.discussion_state = 'visible'
AND p.post_id > #{last_post_id}
ORDER BY p.post_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if results.empty?
last_post_id = results.last[:id]
next if all_records_exist?(:posts, results.map { |r| r[:id] })
create_posts(results, total: total_posts, offset: processed) do |m|
skip = false
mapped = {}
mapped[:id] = m[:id]
mapped[:user_id] = user_id_from_imported_user_id(m[:user_id]) || Discourse::SYSTEM_USER_ID
mapped[:raw] = process_xenforo_post(m[:raw], m[:id])
mapped[:created_at] = Time.zone.at(m[:created_at])
if m[:id] == m[:first_post_id]
cat_id = category_id_from_imported_category_id(m[:category_id]) || @category_mappings[m[:category_id]] || SiteSetting.uncategorized_category_id
mapped[:category] = cat_id
mapped[:title] = CGI.unescapeHTML(m[:title])
mapped[:views] = m[:view_count]
tags = fetch_thread_tags(m[:topic_id])
mapped[:tags] = tags if tags.present?
mapped[:post_create_action] = proc do |pp|
Permalink.find_or_create_by(url: "threads/#{m[:topic_id]}", topic_id: pp.topic_id)
end
else
parent = topic_lookup_from_imported_post_id(m[:first_post_id])
if parent
mapped[:topic_id] = parent[:topic_id]
else
skip = true
end
end
skip ? nil : mapped
end
processed += results.size
end
end
def fetch_thread_tags(thread_id)
sql = "SELECT t.tag FROM #{TABLE_PREFIX}tag_content tc INNER JOIN #{TABLE_PREFIX}tag t ON tc.tag_id = t.tag_id WHERE tc.content_type = 'thread' AND tc.content_id = #{thread_id.to_i}"
rows = mysql_query(sql).to_a
rows.map { |r| DiscourseTagging.clean_tag(r[:tag]) }.compact
end
# =========================================================================
# 6. XenForo Resource Manager (XFRM)
# =========================================================================
# =========================================================================
# 6. XenForo Resource Manager (XFRM)
# =========================================================================
def import_xfrm
puts "", "--> Importiere XenForo Resource Manager (XFRM)..."
has_xfrm = mysql_query("SHOW TABLES LIKE '#{TABLE_PREFIX}rm_resource'").to_a.present?
unless has_xfrm
puts " Keine XFRM-Tabellen gefunden, überspringe."
return
end
xfrm_cat = Category.find_by_name("Resources") || Category.create!(name: "Resources", user_id: Discourse::SYSTEM_USER_ID)
resources = mysql_query(
"SELECT r.resource_id, r.title, r.tag_line, r.user_id, r.resource_category_id, r.resource_date, r.discussion_thread_id,
t.first_post_id, u.message AS update_message, p.message AS post_message
FROM #{TABLE_PREFIX}rm_resource r
LEFT JOIN #{TABLE_PREFIX}thread t ON t.thread_id = r.discussion_thread_id
LEFT JOIN #{TABLE_PREFIX}post p ON p.post_id = t.first_post_id
LEFT JOIN #{TABLE_PREFIX}rm_resource_update u ON u.resource_update_id = r.description_update_id
WHERE r.resource_state = 'visible'"
).to_a
puts " Gefunden #{resources.size} XFRM-Ressourcen."
resources.each do |res|
raw_msg = res[:update_message].presence || res[:post_message].presence
body_text = raw_msg.present? ? process_xenforo_post(raw_msg, res[:first_post_id].to_i) : ""
tagline = res[:tag_line].present? ? CGI.unescapeHTML(res[:tag_line]) : ""
raw = ""
raw += "**#{tagline}**\n\n" if tagline.present?
raw += body_text.present? ? body_text : "*(Keine Beschreibung angegeben)*"
# --- NEU: HOLE TATSÄCHLICHE RESSOURCENDATEIEN (ZIPS, PDFS) ---
# 'v.version_id' auf 'v.resource_version_id' in XenForo 2 aktualisiert
version_files_sql = "
SELECT a.attachment_id, a.data_id, d.filename, d.file_hash, d.user_id
FROM #{TABLE_PREFIX}attachment a
INNER JOIN #{TABLE_PREFIX}attachment_data d ON a.data_id = d.data_id
INNER JOIN #{TABLE_PREFIX}rm_resource_version v ON v.resource_version_id = a.content_id
WHERE a.content_type = 'resource_version' AND v.resource_id = #{res[:resource_id]}
"
resource_files = mysql_query(version_files_sql).to_a
if resource_files.any?
raw += "\n\n### Herunterladbare Dateien\n"
resource_files.each do |file|
upload = import_xf_attachment_file(file[:data_id], file[:file_hash], file[:user_id], file[:filename])
if upload&.persisted?
html = @uploader.html_for_upload(upload, file[:filename])
raw += "\n* #{html}"
end
end
end
version_urls_sql = "
SELECT version_string, download_url
FROM #{TABLE_PREFIX}rm_resource_version
WHERE resource_id = #{res[:resource_id]} AND download_url != ''
"
external_urls = mysql_query(version_urls_sql).to_a
if external_urls.any?
raw += "\n\n### Externe Downloads\n"
external_urls.each do |ext|
raw += "\n* [Version #{ext[:version_string]}](#{ext[:download_url]})"
end
end
# --- ENDE NEUE DATEI-ABFRAGE ---
if res[:discussion_thread_id].to_i > 0 && res[:first_post_id].to_i > 0
topic_map = topic_lookup_from_imported_post_id(res[:first_post_id].to_i)
if topic_map
raw += "\n\n---\n*Original-Diskussionsthread: [Hier ansehen](/t/-/#{topic_map[:topic_id]})*"
end
end
user_id = user_id_from_imported_user_id(res[:user_id]) || Discourse::SYSTEM_USER_ID
create_post = {
id: "xfrm_v5_#{res[:resource_id]}", # Auf v5 erhöht, um den abgestürzten Cache zu leeren
user_id: user_id,
category: xfrm_cat.id,
title: CGI.unescapeHTML(res[:title]),
raw: raw,
created_at: Time.zone.at(res[:resource_date])
}
create_posts([create_post], total: resources.size, offset: 0) { |p| p }
end
end
# =========================================================================
# 7. XenForo Media Gallery (XFMG)
# =========================================================================
def import_xfmg
puts "", "--> Importiere XenForo Media Gallery (XFMG)..."
has_xfmg = mysql_query("SHOW TABLES LIKE '#{TABLE_PREFIX}mg_media_item'").to_a.present?
unless has_xfmg
puts " Keine XFMG-Tabellen gefunden, überspringe."
return
end
xfmg_cat = Category.find_by_name("Media Gallery") || Category.create!(name: "Media Gallery", user_id: Discourse::SYSTEM_USER_ID)
items = mysql_query(
"SELECT m.media_id, m.title, m.description, m.media_type, m.media_tag, m.user_id, m.media_date
FROM #{TABLE_PREFIX}mg_media_item m
WHERE m.media_state = 'visible'"
).to_a
puts " Gefunden #{items.size} XFMG-Medienobjekte."
items.each do |item|
raw = item[:description].presence || ""
# Wenn es ein eingebettetes Video ist (YouTube, Vimeo usw.), die Embed-URL einfügen
if item[:media_type] == "embed" && item[:media_tag].present?
raw = "#{item[:media_tag]}\n\n#{raw}"
else
# Bildanhänge mit XF 2.x 'xfmg_media' content_type verarbeiten
raw = process_xf_attachments("xfmg_media", raw, item[:media_id])
end
raw = "*(Kein Medieninhalt)*" if raw.blank?
user_id = user_id_from_imported_user_id(item[:user_id]) || Discourse::SYSTEM_USER_ID
post_data = {
id: "xfmg_#{item[:media_id]}",
user_id: user_id,
category: xfmg_cat.id,
title: CGI.unescapeHTML(item[:title].presence || "Media #{item[:media_id]}"),
raw: raw,
created_at: Time.zone.at(item[:media_date])
}
create_posts([post_data], total: items.size, offset: 0) { |p| p }
end
end
# =========================================================================
# 8. Private Nachrichten (Unterhaltungen)
# =========================================================================
def import_private_messages
puts "", "--> Importiere Private Nachrichten (Unterhaltungen)..."
last_conv_id = 0
total_count = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}conversation_master").to_a.first[:count]
puts " Gefunden #{total_count} private Nachrichten-Unterhaltungen."
processed = 0
loop do
posts = mysql_query(
"SELECT c.conversation_id, c.title, m.message_id, m.message, m.user_id, m.message_date, c.first_message_id
FROM #{TABLE_PREFIX}conversation_master c
INNER JOIN #{TABLE_PREFIX}conversation_message m ON m.conversation_id = c.conversation_id
WHERE c.conversation_id > #{last_conv_id}
ORDER BY c.conversation_id ASC, m.message_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if posts.empty?
last_conv_id = posts.last[:conversation_id]
next if all_records_exist?(:posts, posts.map { |p| "pm_#{p[:message_id]}" })
create_posts(posts, total: total_count, offset: processed) do |post|
user_id = user_id_from_imported_user_id(post[:user_id]) || Discourse::SYSTEM_USER_ID
message_id = "pm_#{post[:message_id]}"
raw = process_xenforo_post(post[:message], 0)
next if raw.blank?
msg = {
id: message_id,
user_id: user_id,
raw: raw,
created_at: Time.zone.at(post[:message_date].to_i),
import_mode: true
}
if post[:message_id] == post[:first_message_id]
msg[:title] = CGI.unescapeHTML(post[:title])
msg[:archetype] = Archetype.private_message
recipients_sql = "SELECT user_id FROM #{TABLE_PREFIX}conversation_recipient WHERE conversation_id = #{post[:conversation_id].to_i}"
recip_rows = mysql_query(recipients_sql).to_a
recip_user_ids = recip_rows.map { |r| user_id_from_imported_user_id(r[:user_id]) }.compact
target_usernames = User.where(id: recip_user_ids).pluck(:username)
if target_usernames.blank?
fallback_user = User.find_by(id: Discourse::SYSTEM_USER_ID) || User.admins.first
target_usernames = [fallback_user.username] if fallback_user
end
msg[:target_usernames] = target_usernames.join(",")
else
first_msg_map = topic_lookup_from_imported_post_id("pm_#{post[:first_message_id]}")
if first_msg_map
msg[:topic_id] = first_msg_map[:topic_id]
else
next
end
end
msg
end
processed += posts.size
if processed % 5000 == 0
ActiveRecord::Base.connection.clear_query_cache
GC.start
end
end
end
# =========================================================================
# 9. Reaktionen & Likes
# =========================================================================
def import_likes
puts "", "--> Leere kontaminierte Likes und setze Caches aus vorherigen Versuchen zurück..."
# Rausche Aktionen löschen
PostAction.where(post_action_type_id: PostActionType.types[:like]).delete_all
# Gezchte Zähler in der UI auf Null setzen
Post.update_all(like_count: 0)
UserStat.update_all(likes_given: 0, likes_received: 0)
puts "--> Importiere XenForo 2.3 Beitrag-Reaktionen (Likes)..."
# Zählen mit values.first für Treiberübergreifende Sicherheit
total_count = mysql_query(
"SELECT COUNT(*) FROM #{TABLE_PREFIX}reaction_content WHERE content_type = 'post' AND is_counted = 1"
).first.values.first
puts " Gefunden #{total_count} aktive Beitrag-Reaktionen."
last_reaction_content_id = 0
processed = 0
loop do
# Sichere Pagination mit dem Primärschlüssel (reaction_content_id)
reactions = mysql_query(<<-SQL).to_a
SELECT reaction_content_id, content_id, reaction_user_id, reaction_date
FROM #{TABLE_PREFIX}reaction_content
WHERE content_type = 'post'
AND is_counted = 1
AND reaction_content_id > #{last_reaction_content_id}
ORDER BY reaction_content_id ASC
LIMIT #{BATCH_SIZE}
SQL
break if reactions.empty?
# Sichere Extraktion des höchsten Primärschlüssels für das nächste Paginations-Batch
last_row = reactions.last
last_reaction_content_id = (last_row[:reaction_content_id] || last_row["reaction_content_id"]).to_i
create_likes(reactions, total: total_count, offset: processed) do |row|
xf_post_id = row[:content_id] || row["content_id"]
xf_user_id = row[:reaction_user_id] || row["reaction_user_id"]
xf_date = row[:reaction_date] || row["reaction_date"]
# ROHE XENFORO-IDS DIREKT ÜBERGEBEN
# Discourses `create_likes`-Helfer übersetzt diese automatisch im Hintergrund.
{
post_id: xf_post_id,
user_id: xf_user_id,
created_at: Time.zone.at(xf_date.to_i)
}
end
processed += reactions.size
if processed % 10000 == 0
ActiveRecord::Base.connection.clear_query_cache
GC.start
end
end
end
# =========================================================================
# 10. BBCode-Parser & Formatierungs-Transformer
# =========================================================================
def process_xenforo_post(raw, import_id)
return "" if raw.blank?
s = raw.dup
s = s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "") unless s.valid_encoding?
s.gsub!(%r{<!-- s(\S+) --><img (?:[^>]+) /><!-- s(?:\S+) -->}, '\1')
s.gsub!(%r{<!-- \w --><a(?:.+)href="(\S+)"(?:.*)>(.+)</a><!-- \w -->}, '[\2](\1)')
s.gsub!(/:(?:\w{8})\]/, "]")
s = CGI.unescapeHTML(s)
s.gsub!(/\[QUOTE="?([^,\]]+)(?:,\s*post:\s*(\d+))?(?:,\s*member:\s*\d+)"?\)]/i) do
username = $1
imported_post_id = $2
if imported_post_id.present?
topic_mapping = topic_lookup_from_imported_post_id(imported_post_id.to_i)
if topic_mapping
"\n[quote=\"#{username}, post:#{topic_mapping[:post_number]}, topic:#{topic_mapping[:topic_id]}\"]\n"
else
"\n[quote=\"#{username}\"]\n"
end
else
"\n[quote=\"#{username}\"]\n"
end
end
s.gsub!(%r{\[/QUOTE\]}i, "\n[/quote]\n")
s.gsub!(%r{\[HEADING=1\](.+?)\[/HEADING\]}i) { "\n# #{$1}\n" }
s.gsub!(%r{\[HEADING=2\](.+?)\[/HEADING\]}i) { "\n## #{$1}\n" }
s.gsub!(%r{\[HEADING=3\](.+?)\[/HEADING\]}i) { "\n### #{$1}\n" }
s.gsub!(%r{\[SPOILER="?([^\]]*?)"?\](.*?)\[/SPOILER\]}im) do
title = $1.presence || "Spoiler"
content = $2
"\n[details=\"#{title}\"]\n#{content}\n[/details]\n"
end
s.gsub!(%r{\[CODE="?([a-zA-Z0-9_\-+]*)"?\](.*?)\[/CODE\]}im) do
lang = $1.presence || ""
code = $2
"\n```#{lang}\n#{code}\n```\n"
end
s.gsub!(%r{\[USER=\d+\]@?(.+?)\[/USER\]}i, '@\1')
s.gsub!(%r{\[MEDIA=youtube\](.+?)\[/MEDIA\]}i, 'https://www.youtube.com/watch?v=\1')
s.gsub!(%r{\[MEDIA=[^\]]+\](.+?)\[/MEDIA\]}i, '\1')
# Präsentative BBCode-Tags entfernen, die in Markdown nicht verwendet werden
s.gsub!(%r{\[/?(?:LEFT|RIGHT|CENTER|JUSTIFY|FONT|SIZE|COLOR|INDENT)(?:=[^\]]+)?\]}i, '')
# [ATTACH]-Tags verarbeiten und durch Discourse-Uploads ersetzen
s = process_xf_attachments("post", s, import_id) if import_id.to_i > 0
s.strip
end
# =========================================================================
# 11. Anhangauflösung und Dateiverarbeitung
# =========================================================================
def process_xf_attachments(content_type, text, content_id)
sql = "
SELECT a.attachment_id, a.data_id, d.filename, d.file_hash, d.user_id
FROM #{TABLE_PREFIX}attachment a
INNER JOIN #{TABLE_PREFIX}attachment_data d ON a.data_id = d.data_id
WHERE a.content_type = '#{@client.escape(content_type.to_s)}'
AND a.content_id = #{content_id.to_i}
"
attachments = mysql_query(sql).to_a
return text if attachments.empty?
embedded_ids = Set.new
attachments.each do |att|
att_id = att[:attachment_id]
upload = import_xf_attachment_file(att[:data_id], att[:file_hash], att[:user_id], att[:filename])
next unless upload&.persisted?
html = @uploader.html_for_upload(upload, att[:filename])
# Standardmäßige [ATTACH]123[/ATTACH] sowie XF 2.3-Attributvarianten wie [ATTACH type="full" alt="..."]123[/ATTACH] abgleichen
pattern = %r{\[ATTACH[^\]]*\]\s*#{att_id}\s*\[/ATTACH\]}i
if text.match?(pattern)
text.gsub!(pattern, "\n#{html}\n")
embedded_ids.add(att_id)
end
end
# Anhänge hinzufügen, die nicht inline im Textkörper platziert wurden
unattached = attachments.reject { |a| embedded_ids.include?(a[:attachment_id]) }
if unattached.any?
text += "\n\n"
unattached.each do |att|
upload = import_xf_attachment_file(att[:data_id], att[:file_hash], att[:user_id], att[:filename])
next unless upload&.persisted?
html = @uploader.html_for_upload(upload, att[:filename])
text += "#{html}\n"
end
end
# Letzter Bereinigungsdurchlauf: Verwaiste oder fehlende [ATTACH]-Markups entfernen
text.gsub!(%r{\[ATTACH[^\]]*\]\s*\d*\s*\[/ATTACH\]}i, '')
text
end
def import_xf_attachment_file(data_id, file_hash, owner_id, original_filename)
group_id = data_id.to_i / 1000
exact_name = "#{data_id}-#{file_hash}.data"
path_grouped = Pathname.new(File.join(ATTACHMENT_DIR, group_id.to_s, exact_name))
path_flat = Pathname.new(File.join(ATTACHMENT_DIR, exact_name))
path = if File.exist?(path_grouped)
path_grouped
elsif File.exist?(path_flat)
path_flat
else
glob_grouped = Dir.glob(File.join(ATTACHMENT_DIR, group_id.to_s, "#{data_id}-*.data")).first
glob_flat = Dir.glob(File.join(ATTACHMENT_DIR, "#{data_id}-*.data")).first
found = glob_grouped || glob_flat
if found
Pathname.new(found)
else
log_missing_file(:attachment,
data_id: data_id,
filename: original_filename,
db_hash: file_hash,
expected_grouped: path_grouped.to_s,
expected_flat: path_flat.to_s
)
return nil
end
end
discourse_user_id = user_id_from_imported_user_id(owner_id) || Discourse::SYSTEM_USER_ID
temp_path = path.dirname.join(original_filename)
FileUtils.cp(path, temp_path)
upload = create_upload(discourse_user_id, temp_path, original_filename)
FileUtils.rm(temp_path) if File.exist?(temp_path)
upload
rescue StandardError => e
STDERR.puts "Fehler bei der Verarbeitung des Anhangs mit data_id #{data_id}: #{e.message}"
nil
end
def mysql_query(sql)
@client.query(sql)
end
end
ImportScripts::XenForo23.new.perform
Bei einigen Benutzern ist der Import fehlgeschlagen, da durch Spamfilter seltsame E-Mail-Adressen erstellt wurden. Zum Beispiel Adressen wie Peter.Today.Rain.Import.Something@Dave.Gmail.com.
Ich werde meinen oben genannten Importcode aktualisieren/bearbeiten, sobald die benutzerdefinierten Felder einfließen, da diese für mich wichtig sind.
.
