Ich habe viel Code refaktoriert. Dies scheint derzeit ziemlich gut zu funktionieren. Die benutzerdefinierten Felder, die du vielleicht hast, werden nicht übernommen, aber ich habe das wieder neu geschrieben (siehe unten) und werde meinen Import bereinigen und von vorne beginnen.
Bestätigte Funktionstüchtigkeit von Xenforo 2.3 auf selbst gehostetem Discourse:
Benutzer
Benutzerdefinierte Benutzerfelder
PN / Unterhaltungen
XenForo Media Gallery
Xenforo Resource Manager
Likes und Reaktionen
Bearbeitung von Duplikaten und ungültigen E-Mails
XF2.3 BB-Code-Verarbeitung
XFMedia Gallery
XF Resources
Ich habe dies mit 750.000 Beiträgen, 20.000 Mitgliedern und 20,7 GB Anhängen getestet. Der Import dauert etwa 20 Stunden. Nach dem Import benötigt es noch ein paar Stunden, bis Sidekiq aufgeholt hat.
Hinweise.
Ich musste in CSF erlauben, dass mein Docker auf die lokale DB zugreifen kann.
Die Mounts sind wichtig:
ATTACHMENT_DIR = ENV["XF_ATTACHMENT_DIR"] || "/shared/import/internal_data/attachments"
AVATAR_DIR = ENV["XF_AVATAR_DIR"] || "/shared/import/data/avatars"
Dies spart dir, die Daten in die Docker-Instanz zu verschieben. Du verknüpfst hier
# frozen_string_literal: true
# XenForo 2.3 zu Discourse Import-Skript
# Speziell für TurboRenault (dev.turborenault.co.uk) konfiguriert
# Unterstützt:
# - 750.000+ Beiträge mit Keyset-Paginierung (O(1) schnelles Batching)
# - 20,7 GB Anhänge & Avatare (XF 2.3 Pfadauflösung)
# - Benutzerdefinierte Profilfelder & Benutzerdefinierte Themen-/Beitragfelder
# - Benutzergruppen & Sekundäre Gruppenmitgliedschaften (Kollisionsfreie 20-Zeichen-Trunkierung)
# - Private Nachrichten / Unterhaltungen & Mehrere Empfänger (Mit Fallback für gelöschte Benutzer)
# - XenForo Media Gallery (XFMG)
# - XenForo Resource Manager (XFRM mit genauer Auflösung von Themen und Updates)
# - Likes & Reaktionen
# - Automatische Fallback-Verarbeitung für Duplikate & ungültige E-Mails
# - XF 2.3 BBCode / Formatierungs-Parser
#
# Ausführung:
# cd /var/discourse
# ./launcher enter turborenault
# su - discourse
# cd /var/www/discourse
# RAILS_ENV=production bundle exec ruby /shared/xenforo23.rb
require "mysql2"
require "set"
# Robuste Pfadauflösung für base.rb
base_path = File.expand_path("../base.rb", __FILE__)
base_path = "/var/www/discourse/script/import_scripts/base.rb" unless File.exist?(base_path)
require base_path
class ImportScripts::XenForo23 < ImportScripts::Base
# Vorausgefüllte Datenbank-Anmeldedaten
XENFORO_DB = ENV["XF_DB_NAME"] || "XenTR"
DB_HOST = ENV["XF_DB_HOST"] || "172.17.0.1"
DB_PORT = (ENV["XF_DB_PORT"] || 3306).to_i
DB_USER = ENV["XF_DB_USER"] || "daddy"
DB_PASS = ENV["XF_DB_PASS"] || "ABC123abc"
TABLE_PREFIX = ENV["XF_TABLE_PREFIX"] || "xf_"
BATCH_SIZE = (ENV["XF_BATCH_SIZE"] || 2000).to_i
# Eingebundene Pfade
ATTACHMENT_DIR = ENV["XF_ATTACHMENT_DIR"] || "/shared/import/internal_data/attachments"
AVATAR_DIR = ENV["XF_AVATAR_DIR"] || "/shared/import/data/avatars"
# Funktions-Schalter
IMPORT_PM = ENV.fetch("IMPORT_PM", "true") == "true"
IMPORT_LIKES = ENV.fetch("IMPORT_LIKES", "true") == "true"
IMPORT_XFMG = ENV.fetch("IMPORT_XFMG", "true") == "true"
IMPORT_XFRM = ENV.fetch("IMPORT_XFRM", "true") == "true"
IMPORT_CUSTOM_FIELDS = ENV.fetch("IMPORT_CUSTOM_FIELDS", "true") == "true"
def initialize
super
puts "--> Verbinde mit MySQL-Datenbank '#{XENFORO_DB}' auf #{DB_HOST}:#{DB_PORT} als '#{DB_USER}'..."
@client = Mysql2::Client.new(
host: DB_HOST,
port: DB_PORT,
username: DB_USER,
password: DB_PASS,
database: XENFORO_DB,
symbolize_keys: true,
encoding: "utf8mb4",
reconnect: true
)
@category_mappings = {}
@group_mappings = {}
@user_custom_fields_map = {}
@seen_emails = Set.new
@missing_files_count = 0
end
def execute
optimize_discourse_for_import
import_custom_user_field_definitions if IMPORT_CUSTOM_FIELDS
import_users
import_groups
import_categories
import_posts
import_xfrm if IMPORT_XFRM
import_xfmg if IMPORT_XFMG
import_private_messages if IMPORT_PM
import_likes if IMPORT_LIKES
restore_discourse_settings
report_missing_files_summary
end
# =========================================================================
# Leistungsoptimierung für modernes Discourse
# =========================================================================
def optimize_discourse_for_import
puts "", "--> Wende Discourse-Hochgeschwindigkeits-Importoptimierungen an..."
begin
SiteSetting.disable_emails = "all" if SiteSetting.respond_to?(:disable_emails=)
SiteSetting.process_with_fastimage = false if SiteSetting.respond_to?(:process_with_fastimage=)
SiteSetting.max_topic_title_length = 500 if SiteSetting.respond_to?(:max_topic_title_length=)
SiteSetting.min_topic_title_length = 1 if SiteSetting.respond_to?(:min_topic_title_length=)
SiteSetting.min_post_length = 1 if SiteSetting.respond_to?(:min_post_length=)
SiteSetting.allow_duplicate_topic_titles = true if SiteSetting.respond_to?(:allow_duplicate_topic_titles=)
RateLimiter.disable rescue nil
rescue StandardError => e
puts " Optimierungs-Hinweis: #{e.message}"
end
end
def report_missing_files_summary
if @missing_files_count > 0
puts "", " *** #{@missing_files_count} Dateien wurden während des Imports NICHT auf der Festplatte gefunden ***"
else
puts " Alle Anhang-/Avatar-Dateien wurden in Ordnung gefunden."
end
end
def log_missing_file(type, context = {})
@missing_files_count += 1
label = type.to_s.upcase
details = context.map { |k, v| " #{k}: #{v}" }.join("\n")
puts "\n [FEHLT #{label}]\n#{details}"
end
def restore_discourse_settings
puts "", "--> Stellt Standard-Discourse-Siteeinstellungen wieder her..."
begin
SiteSetting.disable_emails = "non_staff" if SiteSetting.respond_to?(:disable_emails=)
RateLimiter.enable rescue nil
rescue StandardError
# ignorieren
end
puts "--> Import erfolgreich abgeschlossen!"
end
# =========================================================================
# 1. Benutzerdefinierte Profilfelder
# =========================================================================
def import_custom_user_field_definitions
puts "", "--> Importiere Definitionen benutzerdefinierter Benutzerfelder..."
fields = mysql_query("SELECT field_id, field_type FROM #{TABLE_PREFIX}user_field").to_a
phrases = {}
begin
phrase_rows = mysql_query("SELECT title, phrase_text FROM #{TABLE_PREFIX}phrase WHERE title LIKE 'user_field_title.%' AND language_id = 1").to_a
phrase_rows = mysql_query("SELECT title, phrase_text FROM #{TABLE_PREFIX}phrase WHERE title LIKE 'user_field_title.%'") if phrase_rows.empty?
phrase_rows.each do |p|
fid = p[:title].to_s.sub("user_field_title.", "")
phrases[fid] = p[:phrase_text]
end
rescue StandardError => e
puts " Phrase-Hinweis: #{e.message}"
end
fields.each do |row|
field_id = row[:field_id]
title = phrases[field_id].presence || field_id.to_s.tr("_", " ").split.map(&:capitalize).join(" ")
desc = "Benutzerdefiniertes Profilfeld: #{title}"
field_type = case row[:field_type]
when "textbox", "textarea" then "text"
when "select", "radio" then "dropdown"
when "checkbox" then "confirm"
else "text"
end
uf = UserField.find_or_create_by!(name: title) do |f|
f.description = desc
f.field_type = field_type
f.editable = true
f.show_on_profile = true
f.show_on_user_card = true
end
@user_custom_fields_map[field_id] = uf.id
end
puts " #{@user_custom_fields_map.size} Definitionen benutzerdefinierter Profilfelder zugeordnet."
end
def fetch_user_custom_fields(xf_user_id)
return {} unless IMPORT_CUSTOM_FIELDS && @user_custom_fields_map.present?
sql = "SELECT field_id, field_value FROM #{TABLE_PREFIX}user_field_value WHERE user_id = #{xf_user_id.to_i}"
rows = mysql_query(sql).to_a
return {} if rows.empty?
custom_fields = {}
rows.each do |row|
discourse_field_id = @user_custom_fields_map[row[:field_id]]
next unless discourse_field_id
val = row[:field_value]
if val.to_s.start_with?("a:", "s:", "i:", "b:")
begin
val = PHP.unserialize(val)
val = val.values.join(", ") if val.is_a?(Hash)
val = val.join(", ") if val.is_a?(Array)
rescue StandardError
# so belassen
end
elsif val.to_s.start_with?("[", "{")
begin
parsed = JSON.parse(val)
val = parsed.is_a?(Array) ? parsed.join(", ") : val
rescue JSON::ParserError
# so belassen
end
end
custom_fields["user_field_#{discourse_field_id}"] = val.to_s if val.present?
end
custom_fields
end
# =========================================================================
# 2. Benutzer & Avatare
# =========================================================================
def import_users
puts "", "--> Importiere Benutzer..."
total_count = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}user WHERE user_state = 'valid' AND is_banned = 0").to_a.first[:count]
puts " #{total_count} gültige Benutzer gefunden."
last_user_id = 0
processed = 0
loop do
sql = "SELECT u.user_id, u.username, u.email, u.custom_title, u.register_date,
u.last_activity, u.user_group_id, u.is_moderator, u.is_admin, u.is_staff, u.avatar_date
FROM #{TABLE_PREFIX}user u
WHERE u.user_state = 'valid' AND u.is_banned = 0 AND u.user_id > #{last_user_id}
ORDER BY u.user_id ASC
LIMIT #{BATCH_SIZE}"
results = mysql_query(sql).to_a
break if results.empty?
last_user_id = results.last[:user_id]
next if all_records_exist?(:users, results.map { |u| u[:user_id] })
create_users(results, total: total_count, offset: processed) do |user|
username = clean_username(user[:username])
next if username.blank?
email = clean_email(user[:email], user[:user_id])
c_fields = fetch_user_custom_fields(user[:user_id])
{
id: user[:user_id],
email: email,
username: username,
title: user[:custom_title],
created_at: Time.zone.at(user[:register_date]),
last_seen_at: Time.zone.at(user[:last_activity]),
moderator: user[:is_moderator] == 1 || user[:is_staff] == 1,
admin: user[:is_admin] == 1,
custom_fields: c_fields,
post_create_action: proc { |u| import_avatar(user[:user_id], user[:avatar_date], u) }
}
end
processed += results.size
end
end
def clean_username(name)
return "" if name.blank?
name = name.tr(" ", "_").gsub(/[^a-zA-Z0-9_\-\.]/, "")
name.first(60)
end
def clean_email(email, user_id)
email_str = email.to_s.strip.downcase
# Prüfe Standardgültigkeit
unless email_str.present? && email_str.include?("@") && email_str.match?(/\A[^@\s]+@[^@\s]+\.[^@\s]+\z/)
return "user_#{user_id}@imported.invalid"
end
# Prüfe auf Duplikate im Datensatz oder in der vorhandenen Discourse UserEmail-Tabelle
if @seen_emails.include?(email_str) || UserEmail.exists?(email: email_str)
prefix = email_str.split("@").first.gsub(/[^a-zA-Z0-9]/, "")
return "duplicate_#{user_id}_#{prefix}@imported.invalid"
end
@seen_emails.add(email_str)
email_str
end
def import_avatar(xf_user_id, avatar_date, imported_user)
return if avatar_date.to_i == 0
group_id = xf_user_id / 1000
filename = File.join(AVATAR_DIR, "l", group_id.to_s, "#{xf_user_id}.jpg")
filename = File.join(AVATAR_DIR, "o", group_id.to_s, "#{xf_user_id}.jpg") unless File.exist?(filename)
unless File.exist?(filename)
log_missing_file(:avatar,
user_id: xf_user_id,
expected_l: File.join(AVATAR_DIR, "l", group_id.to_s, "#{xf_user_id}.jpg"),
expected_o: File.join(AVATAR_DIR, "o", group_id.to_s, "#{xf_user_id}.jpg")
)
return
end
upload = create_upload(imported_user.id, filename, "avatar_#{xf_user_id}.jpg")
return if !upload&.persisted?
imported_user.create_user_avatar if imported_user.user_avatar.nil?
imported_user.user_avatar.update(custom_upload_id: upload.id)
imported_user.update(uploaded_avatar_id: upload.id)
rescue StandardError => e
STDERR.puts "Fehler beim Import des Avatars für Benutzer #{xf_user_id}: #{e.message}"
end
# =========================================================================
# 3. Benutzergruppen & Mitgliedschaften (Kollisionsfreie Trunkierung)
# =========================================================================
def import_groups
puts "", "--> Importiere Benutzergruppen und Mitgliedschaften..."
existing_group_names = Set.new(Group.pluck(:name))
xf_groups = mysql_query("SELECT user_group_id, title FROM #{TABLE_PREFIX}user_group").to_a
xf_groups.each do |g|
xf_gid = g[:user_group_id]
raw_title = CGI.unescapeHTML(g[:title].to_s.strip)
# Überspringe Standard-XF-Systemgruppen (1 = Nicht registriert, 2 = Registriert)
next if [1, 2].include?(xf_gid)
sanitised_name = generate_unique_group_name(raw_title, xf_gid, existing_group_names)
next if sanitised_name.blank?
group = Group.find_by(name: sanitised_name) || Group.create!(
name: sanitised_name,
full_name: raw_title,
visibility_level: Group.visibility_levels[:members]
)
@group_mappings[xf_gid] = group.id
end
puts " #{@group_mappings.size} benutzerdefinierte Benutzergruppen zugeordnet."
last_user_id = 0
loop do
relations = mysql_query(
"SELECT user_id, user_group_id
FROM #{TABLE_PREFIX}user_group_relation
WHERE user_id > #{last_user_id}
ORDER BY user_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if relations.empty?
last_user_id = relations.last[:user_id]
relations.each do |r|
discourse_group_id = @group_mappings[r[:user_group_id]]
next unless discourse_group_id
discourse_user_id = user_id_from_imported_user_id(r[:user_id])
next unless discourse_user_id
GroupUser.find_or_create_by!(
group_id: discourse_group_id,
user_id: discourse_user_id
)
end
end
puts " Gruppenmitgliedschaften erfolgreich importiert."
end
def generate_unique_group_name(title, group_id, existing_names)
base_slug = title.to_s.parameterize(separator: '_')
base_slug = "group_#{group_id}" if base_slug.blank?
candidate = base_slug.slice(0, 20).chomp('_')
counter = 1
while existing_names.include?(candidate)
suffix = "_#{counter}"
max_base_len = 20 - suffix.length
trimmed_base = base_slug.slice(0, max_base_len).chomp('_')
candidate = "#{trimmed_base}#{suffix}"
counter += 1
end
existing_names.add(candidate)
candidate
end
# =========================================================================
# 4. Kategorien (Forenhierarchie)
# =========================================================================
def import_categories
puts "", "--> Importiere Kategorien..."
nodes = mysql_query(
"SELECT n.node_id, n.title, n.description, n.parent_node_id, n.node_name, n.display_order, n.node_type_id
FROM #{TABLE_PREFIX}node n
WHERE n.node_type_id IN ('Category', 'Forum')
ORDER BY n.parent_node_id ASC, n.display_order ASC"
).to_a
top_level = nodes.select { |n| n[:parent_node_id] == 0 }
create_categories(top_level) do |c|
{
id: c[:node_id],
name: CGI.unescapeHTML(c[:title]),
description: c[:description],
position: c[:display_order],
post_create_action: proc do |cat|
Permalink.find_or_create_by(url: "forums/#{c[:node_name]}.#{c[:node_id]}", category_id: cat.id) if c[:node_name].present?
end
}
end
top_ids = Set.new(top_level.map { |c| c[:node_id] })
second_level = nodes.select { |n| top_ids.include?(n[:parent_node_id]) }
create_categories(second_level) do |c|
{
id: c[:node_id],
name: CGI.unescapeHTML(c[:title]),
description: c[:description],
position: c[:display_order],
parent_category_id: category_id_from_imported_category_id(c[:parent_node_id]),
post_create_action: proc do |cat|
Permalink.find_or_create_by(url: "forums/#{c[:node_name]}.#{c[:node_id]}", category_id: cat.id) if c[:node_name].present?
end
}
end
second_ids = Set.new(second_level.map { |c| c[:node_id] })
deeper = nodes.reject { |n| n[:parent_node_id] == 0 || top_ids.include?(n[:node_id]) || second_ids.include?(n[:node_id]) }
deeper.each do |c|
parent = c
while parent && !second_ids.include?(parent[:node_id]) && !top_ids.include?(parent[:node_id])
parent = nodes.find { |n| n[:node_id] == parent[:parent_node_id] }
end
if parent
@category_mappings[c[:node_id]] = category_id_from_imported_category_id(parent[:node_id])
end
end
end
# =========================================================================
# 5. Themen & Beiträge (Live-Zähler für 750.000+ Beiträge)
# =========================================================================
def import_posts
puts "", "--> Importiere Themen und Beiträge (750k Keyset-Paginierung)..."
total_posts = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}post p INNER JOIN #{TABLE_PREFIX}thread t ON p.thread_id = t.thread_id WHERE p.message_state = 'visible' AND t.discussion_state = 'visible'").to_a.first[:count]
puts " Gesamtzahl der zu importierenden Beiträge: #{total_posts}"
last_post_id = 0
processed = 0
loop do
results = mysql_query(
"SELECT p.post_id AS id,
p.thread_id AS topic_id,
t.node_id AS category_id,
t.title AS title,
t.first_post_id AS first_post_id,
t.view_count AS view_count,
t.prefix_id AS prefix_id,
p.user_id AS user_id,
p.message AS raw,
p.post_date AS created_at
FROM #{TABLE_PREFIX}post p
INNER JOIN #{TABLE_PREFIX}thread t ON p.thread_id = t.thread_id
WHERE p.message_state = 'visible'
AND t.discussion_state = 'visible'
AND p.post_id > #{last_post_id}
ORDER BY p.post_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if results.empty?
last_post_id = results.last[:id]
next if all_records_exist?(:posts, results.map { |r| r[:id] })
create_posts(results, total: total_posts, offset: processed) do |m|
skip = false
mapped = {}
mapped[:id] = m[:id]
mapped[:user_id] = user_id_from_imported_user_id(m[:user_id]) || Discourse::SYSTEM_USER_ID
mapped[:raw] = process_xenforo_post(m[:raw], m[:id])
mapped[:created_at] = Time.zone.at(m[:created_at])
if m[:id] == m[:first_post_id]
cat_id = category_id_from_imported_category_id(m[:category_id]) || @category_mappings[m[:category_id]] || SiteSetting.uncategorized_category_id
mapped[:category] = cat_id
mapped[:title] = CGI.unescapeHTML(m[:title])
mapped[:views] = m[:view_count]
tags = fetch_thread_tags(m[:topic_id])
mapped[:tags] = tags if tags.present?
mapped[:post_create_action] = proc do |pp|
Permalink.find_or_create_by(url: "threads/#{m[:topic_id]}", topic_id: pp.topic_id)
end
else
parent = topic_lookup_from_imported_post_id(m[:first_post_id])
if parent
mapped[:topic_id] = parent[:topic_id]
else
skip = true
end
end
skip ? nil : mapped
end
processed += results.size
end
end
def fetch_thread_tags(thread_id)
sql = "SELECT t.tag FROM #{TABLE_PREFIX}tag_content tc INNER JOIN #{TABLE_PREFIX}tag t ON tc.tag_id = t.tag_id WHERE tc.content_type = 'thread' AND tc.content_id = #{thread_id.to_i}"
rows = mysql_query(sql).to_a
rows.map { |r| DiscourseTagging.clean_tag(r[:tag]) }.compact
end
# =========================================================================
# 6. XenForo Resource Manager (XFRM)
# =========================================================================
# =========================================================================
# 6. XenForo Resource Manager (XFRM)
# =========================================================================
def import_xfrm
puts "", "--> Importiere XenForo Resource Manager (XFRM)..."
has_xfrm = mysql_query("SHOW TABLES LIKE '#{TABLE_PREFIX}rm_resource'").to_a.present?
unless has_xfrm
puts " Keine XFRM-Tabellen gefunden, überspringe."
return
end
xfrm_cat = Category.find_by_name("Resources") || Category.create!(name: "Resources", user_id: Discourse::SYSTEM_USER_ID)
resources = mysql_query(
"SELECT r.resource_id, r.title, r.tag_line, r.user_id, r.resource_category_id, r.resource_date, r.discussion_thread_id,
t.first_post_id, u.message AS update_message, p.message AS post_message
FROM #{TABLE_PREFIX}rm_resource r
LEFT JOIN #{TABLE_PREFIX}thread t ON t.thread_id = r.discussion_thread_id
LEFT JOIN #{TABLE_PREFIX}post p ON p.post_id = t.first_post_id
LEFT JOIN #{TABLE_PREFIX}rm_resource_update u ON u.resource_update_id = r.description_update_id
WHERE r.resource_state = 'visible'"
).to_a
puts " #{resources.size} XFRM-Ressourcen gefunden."
resources.each do |res|
raw_msg = res[:update_message].presence || res[:post_message].presence
body_text = raw_msg.present? ? process_xenforo_post(raw_msg, res[:first_post_id].to_i) : ""
tagline = res[:tag_line].present? ? CGI.unescapeHTML(res[:tag_line]) : ""
raw = ""
raw += "**#{tagline}**\n\n" if tagline.present?
raw += body_text.present? ? body_text : "*(Keine Beschreibung angegeben)*"
# --- NEU: HOLE TATSÄCHLICHE RESSOURCENDATEIEN (ZIPS, PDFS) ---
# 'v.version_id' zu 'v.resource_version_id' in XenForo 2 aktualisiert
version_files_sql = "
SELECT a.attachment_id, a.data_id, d.filename, d.file_hash, d.user_id
FROM #{TABLE_PREFIX}attachment a
INNER JOIN #{TABLE_PREFIX}attachment_data d ON a.data_id = d.data_id
INNER JOIN #{TABLE_PREFIX}rm_resource_version v ON v.resource_version_id = a.content_id
WHERE a.content_type = 'resource_version' AND v.resource_id = #{res[:resource_id]}
"
resource_files = mysql_query(version_files_sql).to_a
if resource_files.any?
raw += "\n\n### Herunterladbare Dateien\n"
resource_files.each do |file|
upload = import_xf_attachment_file(file[:data_id], file[:file_hash], file[:user_id], file[:filename])
if upload&.persisted?
html = @uploader.html_for_upload(upload, file[:filename])
raw += "\n* #{html}"
end
end
end
version_urls_sql = "
SELECT version_string, download_url
FROM #{TABLE_PREFIX}rm_resource_version
WHERE resource_id = #{res[:resource_id]} AND download_url != ''
"
external_urls = mysql_query(version_urls_sql).to_a
if external_urls.any?
raw += "\n\n### Externe Downloads\n"
external_urls.each do |ext|
raw += "\n* [Version #{ext[:version_string]}](#{ext[:download_url]})"
end
end
# --- ENDE NEUER DATEI-ABHOLUNG ---
if res[:discussion_thread_id].to_i > 0 && res[:first_post_id].to_i > 0
topic_map = topic_lookup_from_imported_post_id(res[:first_post_id].to_i)
if topic_map
raw += "\n\n---\n*Original-Diskussionsthread: [Hier ansehen](/t/-/#{topic_map[:topic_id]})*"
end
end
user_id = user_id_from_imported_user_id(res[:user_id]) || Discourse::SYSTEM_USER_ID
create_post = {
id: "xfrm_v5_#{res[:resource_id]}", # Auf v5 erhöht, um den abgestürzten Cache zu leeren
user_id: user_id,
category: xfrm_cat.id,
title: CGI.unescapeHTML(res[:title]),
raw: raw,
created_at: Time.zone.at(res[:resource_date])
}
create_posts([create_post], total: resources.size, offset: 0) { |p| p }
end
end
# =========================================================================
# 7. XenForo Media Gallery (XFMG)
# =========================================================================
def import_xfmg
puts "", "--> Importiere XenForo Media Gallery (XFMG)..."
has_xfmg = mysql_query("SHOW TABLES LIKE '#{TABLE_PREFIX}mg_media_item'").to_a.present?
unless has_xfmg
puts " Keine XFMG-Tabellen gefunden, überspringe."
return
end
xfmg_cat = Category.find_by_name("Media Gallery") || Category.create!(name: "Media Gallery", user_id: Discourse::SYSTEM_USER_ID)
items = mysql_query(
"SELECT m.media_id, m.title, m.description, m.media_type, m.media_tag, m.user_id, m.media_date
FROM #{TABLE_PREFIX}mg_media_item m
WHERE m.media_state = 'visible'"
).to_a
puts " #{items.size} XFMG-Medienobjekte gefunden."
items.each do |item|
raw = item[:description].presence || ""
# Wenn es ein eingebettetes Video ist (YouTube, Vimeo, etc.), schließe die Embed-URL ein
if item[:media_type] == "embed" && item[:media_tag].present?
raw = "#{item[:media_tag]}\n\n#{raw}"
else
# Verarbeite Bildanhänge mit XF 2.x 'xfmg_media' content_type
raw = process_xf_attachments("xfmg_media", raw, item[:media_id])
end
raw = "*(Kein Medieninhalt)*" if raw.blank?
user_id = user_id_from_imported_user_id(item[:user_id]) || Discourse::SYSTEM_USER_ID
post_data = {
id: "xfmg_#{item[:media_id]}",
user_id: user_id,
category: xfmg_cat.id,
title: CGI.unescapeHTML(item[:title].presence || "Media #{item[:media_id]}"),
raw: raw,
created_at: Time.zone.at(item[:media_date])
}
create_posts([post_data], total: items.size, offset: 0) { |p| p }
end
end
# =========================================================================
# 8. Private Nachrichten (Unterhaltungen)
# =========================================================================
def import_private_messages
puts "", "--> Importiere Private Nachrichten (Unterhaltungen)..."
last_conv_id = 0
total_count = mysql_query("SELECT COUNT(*) AS count FROM #{TABLE_PREFIX}conversation_master").to_a.first[:count]
puts " #{total_count} private Nachrichtengespräche gefunden."
processed = 0
loop do
posts = mysql_query(
"SELECT c.conversation_id, c.title, m.message_id, m.message, m.user_id, m.message_date, c.first_message_id
FROM #{TABLE_PREFIX}conversation_master c
INNER JOIN #{TABLE_PREFIX}conversation_message m ON m.conversation_id = c.conversation_id
WHERE c.conversation_id > #{last_conv_id}
ORDER BY c.conversation_id ASC, m.message_id ASC
LIMIT #{BATCH_SIZE}"
).to_a
break if posts.empty?
last_conv_id = posts.last[:conversation_id]
next if all_records_exist?(:posts, posts.map { |p| "pm_#{p[:message_id]}" })
create_posts(posts, total: total_count, offset: processed) do |post|
user_id = user_id_from_imported_user_id(post[:user_id]) || Discourse::SYSTEM_USER_ID
message_id = "pm_#{post[:message_id]}"
raw = process_xenforo_post(post[:message], 0)
next if raw.blank?
msg = {
id: message_id,
user_id: user_id,
raw: raw,
created_at: Time.zone.at(post[:message_date].to_i),
import_mode: true
}
if post[:message_id] == post[:first_message_id]
msg[:title] = CGI.unescapeHTML(post[:title])
msg[:archetype] = Archetype.private_message
recipients_sql = "SELECT user_id FROM #{TABLE_PREFIX}conversation_recipient WHERE conversation_id = #{post[:conversation_id].to_i}"
recip_rows = mysql_query(recipients_sql).to_a
recip_user_ids = recip_rows.map { |r| user_id_from_imported_user_id(r[:user_id]) }.compact
target_usernames = User.where(id: recip_user_ids).pluck(:username)
if target_usernames.blank?
fallback_user = User.find_by(id: Discourse::SYSTEM_USER_ID) || User.admins.first
target_usernames = [fallback_user.username] if fallback_user
end
msg[:target_usernames] = target_usernames.join(",")
else
first_msg_map = topic_lookup_from_imported_post_id("pm_#{post[:first_message_id]}")
if first_msg_map
msg[:topic_id] = first_msg_map[:topic_id]
else
next
end
end
msg
end
processed += posts.size
if processed % 5000 == 0
ActiveRecord::Base.connection.clear_query_cache
GC.start
end
end
end
# =========================================================================
# 9. Reaktionen & Likes
# =========================================================================
def import_likes
puts "", "--> Leere verschmutzte Likes und setze Caches aus vorherigen Versuchen zurück..."
# Leere die rohen Aktionen
PostAction.where(post_action_type_id: PostActionType.types[:like]).delete_all
# Setze die zwischengespeicherten Zähler in der UI auf Null
Post.update_all(like_count: 0)
UserStat.update_all(likes_given: 0, likes_received: 0)
puts "--> Importiere XenForo 2.3 Beitragsreaktionen (Likes)..."
# Zähle mit values.first für Treiberunabhängigkeit
total_count = mysql_query(
"SELECT COUNT(*) FROM #{TABLE_PREFIX}reaction_content WHERE content_type = 'post' AND is_counted = 1"
).first.values.first
puts " #{total_count} aktive Beitragsreaktionen gefunden."
last_reaction_content_id = 0
processed = 0
loop do
# Sichere Paginierung mit dem Primärschlüssel (reaction_content_id)
reactions = mysql_query(<<-SQL).to_a
SELECT reaction_content_id, content_id, reaction_user_id, reaction_date
FROM #{TABLE_PREFIX}reaction_content
WHERE content_type = 'post'
AND is_counted = 1
AND reaction_content_id > #{last_reaction_content_id}
ORDER BY reaction_content_id ASC
LIMIT #{BATCH_SIZE}
SQL
break if reactions.empty?
# Sichere Extraktion des höchsten Primärschlüssels für das nächste Paginierungs-Batch
last_row = reactions.last
last_reaction_content_id = (last_row[:reaction_content_id] || last_row["reaction_content_id"]).to_i
create_likes(reactions, total: total_count, offset: processed) do |row|
xf_post_id = row[:content_id] || row["content_id"]
xf_user_id = row[:reaction_user_id] || row["reaction_user_id"]
xf_date = row[:reaction_date] || row["reaction_date"]
# ÜBERGIB ROHE XENFORO-IDS DIREKT
# Discourses `create_likes`-Hilfsfunktion übersetzt diese automatisch im Hintergrund.
{
post_id: xf_post_id,
user_id: xf_user_id,
created_at: Time.zone.at(xf_date.to_i)
}
end
processed += reactions.size
if processed % 10000 == 0
ActiveRecord::Base.connection.clear_query_cache
GC.start
end
end
end
# =========================================================================
# 10. BBCode-Parser & Formatierungstransformer
# =========================================================================
def process_xenforo_post(raw, import_id)
return "" if raw.blank?
s = raw.dup
s = s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "") unless s.valid_encoding?
s.gsub!(%r{<!-- s(\S+) --><img (?:[^>]+) /><!-- s(?:\S+) -->}, '\1')
s.gsub!(%r{<!-- \w --><a(?:.+)href="(\S+)"(?:.*)>(.+)</a><!-- \w -->}, '[\2](\1)')
s.gsub!(/:(?:\w{8})\]/, "]")
s = CGI.unescapeHTML(s)
s.gsub!(/\[QUOTE="?([^,\]]+)(?:,\s*post:\s*(\d+))?(?:,\s*member:\s*\d+)"?\)])/i) do
username = $1
imported_post_id = $2
if imported_post_id.present?
topic_mapping = topic_lookup_from_imported_post_id(imported_post_id.to_i)
if topic_mapping
"\n[quote=\"#{username}, post:#{topic_mapping[:post_number]}, topic:#{topic_mapping[:topic_id]}\"]\n"
else
"\n[quote=\"#{username}\"]\n"
end
else
"\n[quote=\"#{username}\"]\n"
end
end
s.gsub!(%r{\[/QUOTE\]}i, "\n[/quote]\n")
s.gsub!(%r{\[HEADING=1\](.+?)\[/HEADING\]}i) { "\n# #{$1}\n" }
s.gsub!(%r{\[HEADING=2\](.+?)\[/HEADING\]}i) { "\n## #{$1}\n" }
s.gsub!(%r{\[HEADING=3\](.+?)\[/HEADING\]}i) { "\n### #{$1}\n" }
s.gsub!(%r{\[SPOILER="?([^\]]*?)"?\](.*?)\[/SPOILER\]}im) do
title = $1.presence || "Spoiler"
content = $2
"\n[details=\"#{title}\"]\n#{content}\n[/details]\n"
end
s.gsub!(%r{\[CODE="?([a-zA-Z0-9_\-+]*)"?\](.*?)\[/CODE\]}im) do
lang = $1.presence || ""
code = $2
"\n```#{lang}\n#{code}\n```\n"
end
s.gsub!(%r{\[USER=\d+\]@?(.+?)\[/USER\]}i, '@\1')
s.gsub!(%r{\[MEDIA=youtube\](.+?)\[/MEDIA\]}i, 'https://www.youtube.com/watch?v=\1')
s.gsub!(%r{\[MEDIA=[^\]]+\](.+?)\[/MEDIA\]}i, '\1')
# Präsentative BBCode-Tags entfernen, die in Markdown nicht verwendet werden
s.gsub!(%r{\[/?(?:LEFT|RIGHT|CENTER|JUSTIFY|FONT|SIZE|COLOR|INDENT)(?:=[^\]]+)?\]}i, '')
# [ATTACH]-Tags verarbeiten und durch Discourse-Uploads ersetzen
s = process_xf_attachments("post", s, import_id) if import_id.to_i > 0
s.strip
end
# =========================================================================
# 11. Anhangauflösung und Dateiverarbeitung
# =========================================================================
def process_xf_attachments(content_type, text, content_id)
sql = "
SELECT a.attachment_id, a.data_id, d.filename, d.file_hash, d.user_id
FROM #{TABLE_PREFIX}attachment a
INNER JOIN #{TABLE_PREFIX}attachment_data d ON a.data_id = d.data_id
WHERE a.content_type = '#{@client.escape(content_type.to_s)}'
AND a.content_id = #{content_id.to_i}
"
attachments = mysql_query(sql).to_a
return text if attachments.empty?
embedded_ids = Set.new
attachments.each do |att|
att_id = att[:attachment_id]
upload = import_xf_attachment_file(att[:data_id], att[:file_hash], att[:user_id], att[:filename])
next unless upload&.persisted?
html = @uploader.html_for_upload(upload, att[:filename])
# Standard-[ATTACH]123[/ATTACH] sowie XF 2.3-Attributvarianten wie [ATTACH type="full" alt="..."]123[/ATTACH] abgleichen
pattern = %r{\[ATTACH[^\]]*\]\s*#{att_id}\s*\[/ATTACH\]}i
if text.match?(pattern)
text.gsub!(pattern, "\n#{html}\n")
embedded_ids.add(att_id)
end
end
# Anhänge anhängen, die nicht inline im Textkörper platziert wurden
unattached = attachments.reject { |a| embedded_ids.include?(a[:attachment_id]) }
if unattached.any?
text += "\n\n"
unattached.each do |att|
upload = import_xf_attachment_file(att[:data_id], att[:file_hash], att[:user_id], att[:filename])
next unless upload&.persisted?
html = @uploader.html_for_upload(upload, att[:filename])
text += "#{html}\n"
end
end
# Letzter Bereinigungsschritt: Verwaiste oder fehlende [ATTACH]-Markups entfernen
text.gsub!(%r{\[ATTACH[^\]]*\]\s*\d*\s*\[/ATTACH\]}i, '')
text
end
def import_xf_attachment_file(data_id, file_hash, owner_id, original_filename)
group_id = data_id.to_i / 1000
exact_name = "#{data_id}-#{file_hash}.data"
path_grouped = Pathname.new(File.join(ATTACHMENT_DIR, group_id.to_s, exact_name))
path_flat = Pathname.new(File.join(ATTACHMENT_DIR, exact_name))
path = if File.exist?(path_grouped)
path_grouped
elsif File.exist?(path_flat)
path_flat
else
glob_grouped = Dir.glob(File.join(ATTACHMENT_DIR, group_id.to_s, "#{data_id}-*.data")).first
glob_flat = Dir.glob(File.join(ATTACHMENT_DIR, "#{data_id}-*.data")).first
found = glob_grouped || glob_flat
if found
Pathname.new(found)
else
log_missing_file(:attachment,
data_id: data_id,
filename: original_filename,
db_hash: file_hash,
expected_grouped: path_grouped.to_s,
expected_flat: path_flat.to_s
)
return nil
end
end
discourse_user_id = user_id_from_imported_user_id(owner_id) || Discourse::SYSTEM_USER_ID
temp_path = path.dirname.join(original_filename)
FileUtils.cp(path, temp_path)
upload = create_upload(discourse_user_id, temp_path, original_filename)
FileUtils.rm(temp_path) if File.exist?(temp_path)
upload
rescue StandardError => e
STDERR.puts "Fehler bei der Verarbeitung des Anhangs mit data_id #{data_id}: #{e.message}"
nil
end
def mysql_query(sql)
@client.query(sql)
end
end
ImportScripts::XenForo23.new.perform
Bei einigen Nutzern ist die Importierung fehlgeschlagen, da durch Spam-Bots seltsame E-Mail-Adressen erstellt wurden. Zum Beispiel E-Mails wie Peter.Today.Rain.Import.Something@Dave.Gmail.com.
Ich werde meinen oben genannten Import-Code aktualisieren/bearbeiten, sobald die benutzerdefinierten Felder einfließen, da diese für mich wichtig sind.
.
