将 GitLab 从 16.3.9 升级至 16.7.10 版本时后台迁移任务卡住，始终无法完成。
异常表现
在后台迁移中可以看到 merge_request_diffs 迁移始终卡住，然后报错 failed。
检查 sidekiq 日志，可见：
{
"severity": "WARN",
"time": "2026-08-01T04:36:02.119Z",
"retry": 3,
"queue": "default",
"version": 0,
"class": "MigrateExternalDiffsWorker",
"args": [
"3354558"
],
"jid": "0398b26a89255f8a02bd973f",
"created_at": "2026-08-01T03:15:07.865Z",
"meta.caller_id": "ScheduleMigrateExternalDiffsWorker",
"correlation_id": "01KZ5GX8Q8RWFC8SH5GF04V6BP",
"meta.root_caller_id": "Cronjob",
"meta.feature_category": "code_review_workflow",
"meta.client_id": "ip/",
"worker_data_consistency": "always",
"idempotency_key": "resque:gitlab:duplicate:default:2fe6b3ac3aac837a7b182a645d6c1e892a13fe5febf469c83c39eaab326cc4c2",
"size_limiter": "validated",
"enqueued_at": "2026-08-01T04:36:02.022Z",
"retry_count": 2,
"retried_at": "2026-08-01T03:17:22.577Z",
"job_size_bytes": 9,
"pid": 25786,
"message": "MigrateExternalDiffsWorker JID-0398b26a89255f8a02bd973f: fail: 0.136482 sec",
"job_status": "fail",
"scheduling_latency_s": 0,
"redis_calls": 8,
"redis_duration_s": 0.005175,
"redis_read_bytes": 886,
"redis_write_bytes": 605,
"redis_feature_flag_calls": 4,
"redis_feature_flag_duration_s": 0.001378,
"redis_feature_flag_read_bytes": 795,
"redis_feature_flag_write_bytes": 285,
"redis_queues_metadata_calls": 4,
"redis_queues_metadata_duration_s": 0.003797,
"redis_queues_metadata_read_bytes": 91,
"redis_queues_metadata_write_bytes": 320,
"db_count": 7,
"db_write_count": 3,
"db_cached_count": 0,
"db_replica_count": 0,
"db_primary_count": 7,
"db_main_count": 7,
"db_ci_count": 0,
"db_main_replica_count": 0,
"db_ci_replica_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 0,
"db_main_cached_count": 0,
"db_ci_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_main_wal_count": 0,
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_replica_duration_s": 0.0,
"db_primary_duration_s": 0.027,
"db_main_duration_s": 0.027,
"db_ci_duration_s": 0.0,
"db_main_replica_duration_s": 0.0,
"db_ci_replica_duration_s": 0.0,
"external_http_count": 1,
"external_http_duration_s": 0.0295001610647887,
"cpu_s": 0.046835,
"mem_objects": 16084,
"mem_bytes": 6128750,
"mem_mallocs": 4510,
"mem_total_bytes": 6772110,
"worker_id": "sidekiq_3",
"rate_limiting_gates": [],
"duration_s": 0.136482,
"completed_at": "2026-08-01T04:36:02.118Z",
"load_balancing_strategy": "primary",
"exception.class": "ActiveRecord::StatementInvalid",
"exception.message": "PG::CheckViolation: ERROR: new row for relation \"merge_request_diffs\" violates check constraint \"check_11c5f029ad\"\nDETAIL: Failing row contains (3354558, collected, 1607854, 2026-07-28 03:10:31.809121, 2026-08-01 04:36:02.110855, 8cb2b41f2fbe3175c01a9047abef10444a04fd5d, 32, 9a2a3cf431333f367903f736f4770c38c2fa7590, 8cb2b41f2fbe3175c01a9047abef10444a04fd5d, 1, diff-3354558, 1, t, 32, t, 1, null, null).\n",
"exception.backtrace": [
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:141:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:235:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:130:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:61:in `block (2 levels) in \u003cclass:ConnectionProxy\u003e'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:141:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:235:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:130:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:78:in `transaction'",
"app/models/merge_request_diff.rb:580:in `block in migrate_files_to_external_storage!'",
"app/models/concerns/cross_database_modification.rb:92:in `block in transaction'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:141:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:235:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:130:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:78:in `transaction'",
"lib/gitlab/database.rb:359:in `block in transaction'",
"lib/gitlab/database.rb:358:in `transaction'",
"app/models/concerns/cross_database_modification.rb:83:in `transaction'",
"app/models/merge_request_diff.rb:577:in `migrate_files_to_external_storage!'",
"app/services/merge_requests/migrate_external_diffs_service.rb:23:in `execute'",
"app/workers/migrate_external_diffs_worker.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/skip_jobs.rb:49:in `call'",
"lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:29:in `call'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/click_house/migration_support/sidekiq_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb:31:in `perform'",
"lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb:22:in `perform'",
"lib/gitlab/sidekiq_middleware/pause_control/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
"lib/gitlab/application_context.rb:148:in `block in use'",
"lib/gitlab/application_context.rb:148:in `use'",
"lib/gitlab/application_context.rb:80:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:8:in `block in call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:105:in `block in call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:133:in `block in instrument'",
"lib/gitlab/metrics/background_transaction.rb:33:in `run'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:133:in `instrument'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:104:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
],
"exception.cause_class": "PG::CheckViolation",
"exception.sql": "/*application:sidekiq,correlation_id:01KZ5GX8Q8RWFC8SH5GF04V6BP,jid:0398b26a89255f8a02bd973f,endpoint_id:MigrateExternalDiffsWorker,db_config_name:main*/ UPDATE \"merge_request_diffs\" SET \"updated_at\" = $1, \"external_diff\" = $2, \"stored_externally\" = $3 WHERE \"merge_request_diffs\".\"id\" = $4",
"db_duration_s": 0.057793,
"urgency": "low",
"target_duration_s": 300,
"target_scheduling_latency_s": 60
}其中关键信息为 PG::CheckViolation: ERROR: new row for relation \"merge_request_diffs\" violates check constraint \"check_11c5f029ad\"\nDETAIL: Failing row contains
报错原因为数据库 merge_request_diffs 约束条件不满足，登录数据库查询当前约束：
SELECT
conname,
pg_get_constraintdef(oid)
FROM pg_constraint
WHERE conname='check_11c5f029ad';返回：
conname | pg_get_constraintdef
------------------+--------------------------------------------
check_11c5f029ad | CHECK ((project_id IS NOT NULL)) NOT VALID
(1 row)即：“project_id列不能为空”，查询当前为空的记录有多少行：
SELECT count(*) FROM merge_request_diffs WHERE project_id IS NULL;记得先备份数据库以防不测，确认数量后登录数据库：
-- 查询当前超时限制；
SHOW statement_timeout;
-- 如果时间过短，需要调大防止执行失败；
SET statement_timeout = '30min';
-- 查看待修复多少条，和上面的 null 数量是否一致；
SELECT count(*)
FROM merge_request_diffs mrd
JOIN merge_requests mr
ON mrd.merge_request_id = mr.id
WHERE mrd.project_id IS NULL;
-- 检查是否存在无法关联的孤儿 diff，必须为空才能继续执行更新；
SELECT count(*)
FROM merge_request_diffs mrd
LEFT JOIN merge_requests mr
ON mrd.merge_request_id = mr.id
WHERE mrd.project_id IS NULL
AND mr.id IS NULL;
-- 修复所有异常记录
UPDATE merge_request_diffs mrd
SET project_id = mr.target_project_id
FROM merge_requests mr
WHERE mrd.merge_request_id = mr.id
AND mrd.project_id IS NULL;
-- 复查异常记录（返回空即修复完成）
SELECT count(*)
FROM merge_request_diffs
WHERE project_id IS NULL;确认无异常后，重启所有的 sidekiq 服务重新开始后台迁移任务，可以观察到正常开始后台迁移。
附录
参考链接
本文由 柒 创作，采用 知识共享署名4.0
国际许可协议进行许可。
转载本站文章前请注明出处，文章作者保留所有权限。
最后编辑时间： 2026-08-31 04:04 AM