feat(data): 业务数据与系统数据分离的导入导出
数据迁移不再整体替换 trade.db —— 那条路会把账号、权限组、组织、设置（含大模型密钥）、 登录会话与审计日志一起搬到目标部署。改为按类别搬运业务数据，系统数据永远是目标部署自己那份。 - backend/lib/dataTransfer：表目录（30 张业务表，依赖顺序）、gzip JSONL 数据包（每表 sha256）、导入内核 - 归属统一给导入人；冲突跳过不覆盖；父记录按 __refs 自然键快照补壳 - 流水表（price_history）按整行内容判重：用 (sku, changed_at) 当键实测 643 行丢 186 行 - 预览在库副本上真跑一遍导入，报告与真导同源；导入前 VACUUM INTO 备份，单事务写入 - 新权限点 data.transfer（仅超管组），导出/导入均写审计 - 前端「数据迁移」页：按类别勾选导出、上传后预览逐表明细、确认导入；移除设置里的静态占位「数据交换」 - 测试：单测 7 项（含目录外键顺序、BLOB、重复导入、包被改坏）、API 端到端 1 项（两个真实实例互传） - 用真实库副本彩排：3854 行 0 失败，29 张业务表行数全部一致，包 1.22 MB
This commit is contained in:
Drsakura
@@ -0,0 +1,357 @@
|
||||
/**
|
||||
* 数据包（.hdata）的写出与读取。
|
||||
*
|
||||
* 格式：gzip 压缩的 JSONL，三部分
|
||||
* 1. header —— 格式版本、导出时间、导出人、包含的类别与表
|
||||
* 2. 每行一条记录 { "t": "products", "r": { ... } }，BLOB 列以 base64 字符串承载
|
||||
* 3. footer —— 每张表的行数与 sha256
|
||||
*
|
||||
* 为什么不是 zip、为什么不像数据库备份那样直接拷 .db：
|
||||
* - .db 备份是整库，连 users / settings / sessions 一起搬，正是这次要分开的东西；
|
||||
* - zip 需要额外依赖，而 gzip + JSONL 只用 Node 内置能力，还能逐行流式读写
|
||||
* （合同原件是 BLOB，未来单个文件几十 MB，不能整包读进内存）；
|
||||
* - 每张表单独一个 sha256，导入前就能判断包是否被截断 / 传坏，
|
||||
* 而不是解析到一半才炸在某个 JSON 语法错误上。
|
||||
*/
|
||||
const fs = require('fs')
|
||||
const zlib = require('zlib')
|
||||
const crypto = require('crypto')
|
||||
const readline = require('readline')
|
||||
const { Readable } = require('stream')
|
||||
const { pipeline } = require('stream/promises')
|
||||
|
||||
const { TABLES, OWNER_COLUMNS, ORG_COLUMNS, resolveCategories, CATEGORY_LIST } = require('./catalog')
|
||||
|
||||
const FORMAT = 'happy-data'
|
||||
const FORMAT_VERSION = 1
|
||||
const APP_NAME = 'happy出口通'
|
||||
|
||||
/** 单包上限：超过就提示按类别分开导，避免一次把几个 GB 读进内存 */
|
||||
const MAX_BUNDLE_BYTES = 100 * 1024 * 1024
|
||||
const MAX_ROWS = 2_000_000
|
||||
|
||||
/* ------------------------------------------------------------------ 元数据 */
|
||||
|
||||
const quote = name => `"${String(name).replace(/"/g, '""')}"`
|
||||
|
||||
/** 目标库 / 源库的物理结构：列、BLOB 列、主键列 */
|
||||
function tableInfo(db, table) {
|
||||
let columns
|
||||
try {
|
||||
columns = db.pragma(`table_info(${quote(table)})`)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (!columns || !columns.length) return null
|
||||
const pk = columns.find(column => Number(column.pk) === 1) || null
|
||||
return {
|
||||
columns: columns.map(column => ({ name: column.name, type: String(column.type || '').toUpperCase() })),
|
||||
blobs: new Set(columns.filter(column => String(column.type || '').toUpperCase().includes('BLOB')).map(column => column.name)),
|
||||
pk: pk ? pk.name : null,
|
||||
autoId: !!pk && pk.name === 'id'
|
||||
}
|
||||
}
|
||||
|
||||
/** 外键：{ from: 本表列, table: 父表, to: 父表列 } */
|
||||
function foreignKeys(db, table) {
|
||||
let list
|
||||
try {
|
||||
list = db.pragma(`foreign_key_list(${quote(table)})`)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
return (list || []).map(item => ({ from: item.from, table: item.table, to: item.to }))
|
||||
}
|
||||
|
||||
/** 包内是否出现这个表（老库可能缺新表，按实际存在处理） */
|
||||
function tableExists(db, table) {
|
||||
return !!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table)
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ 导出 */
|
||||
|
||||
function encodeValue(value, isBlob) {
|
||||
if (value === null || value === undefined) return null
|
||||
if (isBlob) return Buffer.isBuffer(value) ? value.toString('base64') : Buffer.from(String(value)).toString('base64')
|
||||
if (Buffer.isBuffer(value)) return value.toString('base64')
|
||||
return value
|
||||
}
|
||||
|
||||
function decodeValue(value, isBlob) {
|
||||
if (value === null || value === undefined) return null
|
||||
if (isBlob) return Buffer.from(String(value), 'base64')
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* 把一张表读成「行对象」，并按需附上父记录的自然键快照。
|
||||
*
|
||||
* 为什么要附 `__refs`：只导出订单时，订单里的 customer_id 在目标库里根本不存在。
|
||||
* 有了客户公司名这份快照，导入方就能按公司名找到客户、找不到才补一个空壳，
|
||||
* 而不是整行订单都导不进去。
|
||||
*/
|
||||
function* rowsOfTable(db, table, { info, refs, refCache }) {
|
||||
const columns = info.columns.map(column => column.name)
|
||||
const select = columns.map(quote).join(', ')
|
||||
const orderBy = info.autoId ? 'id' : quote(info.pk || columns[0])
|
||||
const statement = db.prepare(`SELECT ${select} FROM ${quote(table)} ORDER BY ${orderBy}`)
|
||||
|
||||
for (const row of statement.iterate()) {
|
||||
const payload = {}
|
||||
for (const name of columns) {
|
||||
const value = encodeValue(row[name], info.blobs.has(name))
|
||||
/* 空值不写进包里，体积能小一截（products 有 1576 行、不少列是空的） */
|
||||
if (value !== null) payload[name] = value
|
||||
}
|
||||
|
||||
const refValues = {}
|
||||
for (const ref of refs) {
|
||||
const raw = row[ref.from]
|
||||
if (raw === null || raw === undefined) continue
|
||||
const snapshot = refSnapshot(db, ref, raw, refCache)
|
||||
if (snapshot) refValues[ref.from] = snapshot
|
||||
}
|
||||
if (Object.keys(refValues).length) payload.__refs = refValues
|
||||
|
||||
yield payload
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 父记录的自然键快照。
|
||||
* 父表主键本身就是自然键（products.sku）时，值已经在行里，不用再查；
|
||||
* 主键是自增 id（customers / suppliers / product_groups）时按 id 取一次并缓存。
|
||||
*/
|
||||
function refSnapshot(db, ref, value, cache) {
|
||||
if (ref.to && ref.to !== 'id') return value
|
||||
const spec = TABLES.find(item => item.table === ref.table)
|
||||
if (!spec || !spec.key.length) return null
|
||||
const cacheKey = `${ref.table}:${value}`
|
||||
if (cache.has(cacheKey)) return cache.get(cacheKey)
|
||||
let snapshot = null
|
||||
try {
|
||||
const row = db.prepare(`SELECT ${spec.key.map(quote).join(', ')} FROM ${quote(ref.table)} WHERE id = ?`).get(value)
|
||||
if (row) snapshot = spec.key.length === 1 ? row[spec.key[0]] : Object.fromEntries(spec.key.map(key => [key, row[key]]))
|
||||
} catch { /* 父记录已被删：留空，导入时按「找不到父记录」处理 */ }
|
||||
cache.set(cacheKey, snapshot)
|
||||
return snapshot
|
||||
}
|
||||
|
||||
/** 选中类别 → 实际要导出的表（跳过源库里不存在的表） */
|
||||
function planTables(db, categories) {
|
||||
const selected = new Set(resolveCategories(categories))
|
||||
return TABLES.filter(spec => selected.has(spec.category) && tableExists(db, spec.table))
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成数据包的行流（不含 gzip）。
|
||||
* 交给 Readable.from 之后由 gzip 压到 HTTP 响应里 —— 逐行 yield，不整包驻留内存。
|
||||
*/
|
||||
async function* bundleLines(db, { categories, exporter }) {
|
||||
const plans = planTables(db, categories)
|
||||
const header = {
|
||||
type: 'header',
|
||||
format: FORMAT,
|
||||
version: FORMAT_VERSION,
|
||||
app: APP_NAME,
|
||||
exportedAt: new Date().toISOString(),
|
||||
categories: resolveCategories(categories),
|
||||
tableNames: plans.map(spec => spec.table),
|
||||
exporter: exporter ? { id: exporter.id, username: exporter.username, display_name: exporter.display_name } : null
|
||||
}
|
||||
yield JSON.stringify(header) + '\n'
|
||||
|
||||
const tableMeta = {}
|
||||
let totalRows = 0
|
||||
|
||||
for (const spec of plans) {
|
||||
const info = tableInfo(db, spec.table)
|
||||
if (!info) continue
|
||||
const refs = foreignKeys(db, spec.table).filter(ref =>
|
||||
!OWNER_COLUMNS.has(ref.from) && !ORG_COLUMNS.has(ref.from)
|
||||
&& TABLES.some(item => item.table === ref.table))
|
||||
const refCache = new Map()
|
||||
const hash = crypto.createHash('sha256')
|
||||
let count = 0
|
||||
|
||||
for (const payload of rowsOfTable(db, spec.table, { info, refs, refCache })) {
|
||||
const line = JSON.stringify({ t: spec.table, r: payload })
|
||||
hash.update(line)
|
||||
count++
|
||||
yield line + '\n'
|
||||
}
|
||||
|
||||
tableMeta[spec.table] = { rows: count, sha256: hash.digest('hex') }
|
||||
totalRows += count
|
||||
}
|
||||
|
||||
yield JSON.stringify({
|
||||
type: 'footer',
|
||||
version: FORMAT_VERSION,
|
||||
tables: tableMeta,
|
||||
rows: totalRows,
|
||||
categories: header.categories
|
||||
}) + '\n'
|
||||
}
|
||||
|
||||
/** 把数据包写到任意可写流（HTTP 响应 / 文件） */
|
||||
async function writeBundle(db, output, options) {
|
||||
const gzip = zlib.createGzip({ level: 6 })
|
||||
await pipeline(Readable.from(bundleLines(db, options)), gzip, output)
|
||||
}
|
||||
|
||||
/** 导出到文件（命令行工具与测试用） */
|
||||
async function writeBundleFile(db, filePath, options) {
|
||||
await writeBundle(db, fs.createWriteStream(filePath), options)
|
||||
return fs.statSync(filePath).size
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ 读取 */
|
||||
|
||||
function openMaybeGzip(filePath) {
|
||||
const fd = fs.openSync(filePath, 'r')
|
||||
const magic = Buffer.alloc(2)
|
||||
const read = fs.readSync(fd, magic, 0, 2, 0)
|
||||
fs.closeSync(fd)
|
||||
const isGzip = read === 2 && magic[0] === 0x1f && magic[1] === 0x8b
|
||||
return fs.createReadStream(filePath).pipe(isGzip ? zlib.createGunzip() : new (require('stream').PassThrough)())
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取并校验数据包。
|
||||
*
|
||||
* 校验不做「尽力而为」：行数或 sha256 对不上就整体拒绝，并指出是哪张表 ——
|
||||
* 半个包导进去比导不进去更难收拾。
|
||||
*/
|
||||
async function readBundle(filePath) {
|
||||
const stat = fs.statSync(filePath)
|
||||
if (stat.size > MAX_BUNDLE_BYTES) {
|
||||
const error = new Error(`数据包 ${(stat.size / 1024 / 1024).toFixed(1)} MB，超过 ${MAX_BUNDLE_BYTES / 1024 / 1024} MB 上限。请按类别分开导出。`)
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
|
||||
const input = openMaybeGzip(filePath)
|
||||
const reader = readline.createInterface({ input, crlfDelay: Infinity })
|
||||
|
||||
let header = null
|
||||
let footer = null
|
||||
const rows = new Map()
|
||||
const hashes = new Map()
|
||||
const counts = new Map()
|
||||
let lineNo = 0
|
||||
|
||||
for await (const raw of reader) {
|
||||
const line = raw.trim()
|
||||
if (!line) continue
|
||||
lineNo++
|
||||
let parsed
|
||||
try {
|
||||
parsed = JSON.parse(line)
|
||||
} catch {
|
||||
const error = new Error(`数据包第 ${lineNo} 行不是合法 JSON，文件可能已损坏`)
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
|
||||
if (parsed.type === 'header') {
|
||||
if (header) {
|
||||
const error = new Error('数据包出现重复的头部')
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
if (parsed.format !== FORMAT) {
|
||||
const error = new Error('这不是本系统的数据包（缺少 happy-data 标识）')
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
if (Number(parsed.version) > FORMAT_VERSION) {
|
||||
const error = new Error(`数据包版本 ${parsed.version} 高于本系统支持的 ${FORMAT_VERSION}，请先升级系统`)
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
header = parsed
|
||||
continue
|
||||
}
|
||||
|
||||
if (parsed.type === 'footer') {
|
||||
footer = parsed
|
||||
continue
|
||||
}
|
||||
|
||||
if (!header) {
|
||||
const error = new Error('数据包缺少头部，无法识别格式')
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
if (!parsed.t || !parsed.r || typeof parsed.r !== 'object') {
|
||||
const error = new Error(`数据包第 ${lineNo} 行缺少表名或记录内容`)
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
|
||||
if (!rows.has(parsed.t)) {
|
||||
rows.set(parsed.t, [])
|
||||
hashes.set(parsed.t, crypto.createHash('sha256'))
|
||||
counts.set(parsed.t, 0)
|
||||
}
|
||||
const list = rows.get(parsed.t)
|
||||
if (list.length >= MAX_ROWS) {
|
||||
const error = new Error(`数据包内 ${parsed.t} 超过 ${MAX_ROWS} 行，请按类别分开导入`)
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
list.push(parsed.r)
|
||||
counts.set(parsed.t, counts.get(parsed.t) + 1)
|
||||
hashes.get(parsed.t).update(line)
|
||||
}
|
||||
|
||||
if (!header) {
|
||||
const error = new Error('数据包为空或缺少头部')
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
if (!footer) {
|
||||
const error = new Error('数据包缺少结尾清单，可能传输中断。请重新导出后上传。')
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
|
||||
const mismatches = []
|
||||
for (const [table, meta] of Object.entries(footer.tables || {})) {
|
||||
const actualRows = counts.get(table) || 0
|
||||
if (actualRows !== meta.rows) mismatches.push(`${table} 行数应为 ${meta.rows}，实际 ${actualRows}`)
|
||||
else {
|
||||
const actualHash = (hashes.get(table) || crypto.createHash('sha256')).digest('hex')
|
||||
if (actualHash !== meta.sha256) mismatches.push(`${table} 校验值不符`)
|
||||
}
|
||||
}
|
||||
if (mismatches.length) {
|
||||
const error = new Error(`数据包校验失败：${mismatches.slice(0, 4).join('；')}${mismatches.length > 4 ? ` 等 ${mismatches.length} 项` : ''}。请重新导出后上传。`)
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
|
||||
return {
|
||||
header,
|
||||
footer,
|
||||
rows,
|
||||
size: stat.size,
|
||||
categories: (header.categories || []).filter(key => CATEGORY_LIST.some(item => item.key === key))
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
FORMAT,
|
||||
FORMAT_VERSION,
|
||||
MAX_BUNDLE_BYTES,
|
||||
tableInfo,
|
||||
foreignKeys,
|
||||
tableExists,
|
||||
decodeValue,
|
||||
bundleLines,
|
||||
writeBundle,
|
||||
writeBundleFile,
|
||||
planTables,
|
||||
readBundle
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* 业务数据目录 —— 导入 / 导出的唯一事实来源。
|
||||
*
|
||||
* 设计口径（Wayne 2026-10-23 定）：
|
||||
* ① **业务数据与系统数据必须分开**。业务表（客户、订单、产品…）可以整包搬去另一个
|
||||
* 部署；系统表（用户、权限组、组织、会话、设置、审计）**永不随包走** ——
|
||||
* 设置里还存着大模型密钥，会话里是登录态，搬过去等于给别人开后台。
|
||||
* ② **归属一律给导入人**。同事之间快速同步、或职责交接时，「谁导入就归谁」，
|
||||
* 导完再由管理员在界面上重新分配，比在包里猜账号可靠。
|
||||
* ③ **冲突不覆盖**。线上已有的同一条记录（同货号 / 同供应商名 / 同订单号…）跳过并报告，
|
||||
* 绝不静默改掉线上数据。
|
||||
*
|
||||
* 表清单一律按**依赖顺序**排列：父表在前、子表在后。导入时逐行插入，外键始终成立，
|
||||
* 不需要 `PRAGMA foreign_keys=OFF` 这种「先塞进去再祈祷」的做法。
|
||||
* 新增业务表时，只要在 TABLES 里按依赖顺序补一条，导出、导入、界面三处自动跟上。
|
||||
*/
|
||||
|
||||
/** 可选的数据类别：一个类别 = 一组完整的业务闭环，可以单独导出/导入 */
|
||||
const CATEGORY_LIST = [
|
||||
{
|
||||
key: 'catalog',
|
||||
label: '产品目录',
|
||||
description: '供应商、产品与品类、阶梯价、价格历史、包装、产品图片、合同认列记录'
|
||||
},
|
||||
{
|
||||
key: 'customers',
|
||||
label: '客户与往来',
|
||||
description: '客户、联系人、商机、询盘、客户报价单、跟进记录与时间线'
|
||||
},
|
||||
{
|
||||
key: 'orders',
|
||||
label: '订单与支出',
|
||||
description: '订单、订单明细、订单杂费（快递费/吊卡费等）'
|
||||
},
|
||||
{
|
||||
key: 'todos',
|
||||
label: '待办与日程',
|
||||
description: '待办事项与日历日程（不含账号信息）'
|
||||
},
|
||||
{
|
||||
key: 'prospects',
|
||||
label: '新客开发',
|
||||
description: '开发任务与抓取到的线索'
|
||||
}
|
||||
]
|
||||
|
||||
/**
|
||||
* 业务表清单（依赖顺序！）。
|
||||
*
|
||||
* key —— 判定「线上是否已有同一条记录」的列。全部非空才参与去重；
|
||||
* 某列为空时该行按新增处理（宁可多一条，也不误判成重复而丢数据）。
|
||||
* contentKey —— 没有自然业务键的**流水表**（如价格历史）用整行内容比对。
|
||||
* 价格历史里同一 SKU 在同一时间戳可能有多条改价（一次批量导入就会这样），
|
||||
* 拿 (sku, changed_at) 当键会把其中一部分当成重复静默丢掉 —— 实测 643 行丢 186 行。
|
||||
* 整行比对等价于「内容完全相同才算重复」，不会误杀。
|
||||
* stub —— 被别的表引用、但包里没带父记录时，允许用 key[0] 建一条最小记录
|
||||
* （例如导入订单时可以只补一个只有公司名的客户壳）。
|
||||
* label —— 报告里给人看的名字。
|
||||
*
|
||||
* 刻意不在清单里的表：
|
||||
* users / sessions / organizations / permission_groups / permission_group_permissions /
|
||||
* role_permission_groups / settings / audit_log —— 系统数据，永不导出、永不导入。
|
||||
* cart_items —— 只是本机「小推车」的临时挑选，不属于需要迁移的业务数据。
|
||||
*/
|
||||
const TABLES = [
|
||||
/* ---------------- 产品目录 ---------------- */
|
||||
{ table: 'suppliers', label: '供应商', category: 'catalog', key: ['name'], stub: true },
|
||||
{ table: 'product_groups', label: '产品组', category: 'catalog', key: ['name', 'brand'], stub: true },
|
||||
/* 合同是产品价格的来源凭据：products.contract_id 指向它，所以必须排在 products 之前 */
|
||||
{ table: 'contracts', label: '合同认列记录', category: 'catalog', key: ['filename', 'processed_at'] },
|
||||
{ table: 'products', label: '产品', category: 'catalog', key: ['sku'] },
|
||||
{ table: 'supplier_quotes', label: '供应商报价', category: 'catalog', key: ['supplier_id', 'quote_date'] },
|
||||
{ table: 'tier_prices', label: '阶梯价', category: 'catalog', key: ['sku', 'moq_from', 'currency'] },
|
||||
{ table: 'price_history', label: '价格历史', category: 'catalog', contentKey: true },
|
||||
{ table: 'item_attributes', label: '产品属性', category: 'catalog', key: ['sku', 'name'] },
|
||||
{ table: 'product_images', label: '产品图片', category: 'catalog', key: ['sku', 'filename'] },
|
||||
{ table: 'product_image_files', label: '产品图片文件', category: 'catalog', key: ['image_id'] },
|
||||
{ table: 'contract_files', label: '合同原件', category: 'catalog', key: ['contract_id'] },
|
||||
{ table: 'product_packaging', label: '包装规格', category: 'catalog', key: ['sku'] },
|
||||
{ table: 'contract_import_jobs', label: '清洗任务', category: 'catalog', key: ['job_no'] },
|
||||
{ table: 'contract_import_items', label: '清洗明细', category: 'catalog', key: ['job_id', 'filename', 'row_no'] },
|
||||
|
||||
/* ---------------- 客户与往来 ---------------- */
|
||||
{ table: 'customers', label: '客户', category: 'customers', key: ['company'], stub: true },
|
||||
{ table: 'contacts', label: '联系人', category: 'customers', key: ['customer_id', 'name'] },
|
||||
{ table: 'deals', label: '商机', category: 'customers', key: ['customer_id', 'title'] },
|
||||
{ table: 'customer_quotations', label: '客户报价单', category: 'customers', key: ['quote_no'] },
|
||||
{ table: 'quotation_items', label: '报价明细', category: 'customers', key: ['quotation_id', 'sku'] },
|
||||
{ table: 'inquiries', label: '询盘', category: 'customers', key: ['inquiry_no'] },
|
||||
{ table: 'inquiry_items', label: '询盘明细', category: 'customers', key: ['inquiry_id', 'sku'] },
|
||||
{ table: 'follow_up_records', label: '跟进记录', category: 'customers', key: ['customer_id', 'created_at'] },
|
||||
|
||||
/* ---------------- 订单 ---------------- */
|
||||
{ table: 'orders', label: '订单', category: 'orders', key: ['order_no'] },
|
||||
{ table: 'order_items', label: '订单明细', category: 'orders', key: ['order_id', 'sku', 'unit_price'] },
|
||||
{ table: 'order_expenses', label: '订单杂费', category: 'orders', key: ['order_id', 'category', 'amount', 'spent_at'] },
|
||||
|
||||
/* activities 引用 customers / deals / orders 三张表，所以必须排在订单之后 ——
|
||||
否则「时间线关联到哪张订单」这个外键会在导入时解析不到而丢成 NULL。
|
||||
它属于「客户」类别，但依赖顺序优先于类别分组（类别只影响界面勾选）。 */
|
||||
{ table: 'activities', label: '客户时间线', category: 'customers', key: ['customer_id', 'activity_type', 'created_at'] },
|
||||
|
||||
/* ---------------- 待办（todos 引用客户/商机/订单，故排在它们之后） ---------------- */
|
||||
{ table: 'todos', label: '待办与日程', category: 'todos', key: ['title', 'created_at'] },
|
||||
|
||||
/* ---------------- 新客开发 ---------------- */
|
||||
{ table: 'prospect_runs', label: '开发任务', category: 'prospects', key: ['name', 'created_at'] },
|
||||
{ table: 'prospect_leads', label: '开发线索', category: 'prospects', key: ['run_id', 'company'] }
|
||||
]
|
||||
|
||||
/**
|
||||
* 「归属人」类列：导入时一律改写成执行导入的人。
|
||||
* 这些列不随包迁移 —— 包里存的是原机器的用户 id，到这边既无意义也可能撞上别人。
|
||||
*/
|
||||
const OWNER_COLUMNS = new Set(['owner_id', 'created_by', 'assigned_to', 'reviewed_by'])
|
||||
|
||||
/** 归属组织：导入到当前部署的哪个组织（多组织阶段目前只有一个主组织） */
|
||||
const ORG_COLUMNS = new Set(['org_id'])
|
||||
|
||||
const CATEGORY_KEYS = CATEGORY_LIST.map(item => item.key)
|
||||
|
||||
/** 按类别取表（保持依赖顺序） */
|
||||
function tablesOfCategory(category) {
|
||||
return TABLES.filter(item => item.category === category)
|
||||
}
|
||||
|
||||
/** 展开选择：'all' 或省略时返回全部业务表 */
|
||||
function resolveCategories(input) {
|
||||
const list = Array.isArray(input)
|
||||
? input.map(item => String(item || '').trim()).filter(Boolean)
|
||||
: String(input || '').split(',').map(item => item.trim()).filter(Boolean)
|
||||
if (!list.length || list.includes('all')) return CATEGORY_KEYS.slice()
|
||||
const unknown = list.filter(item => !CATEGORY_KEYS.includes(item))
|
||||
if (unknown.length) {
|
||||
const error = new Error(`未知的数据类别：${unknown.join('、')}`)
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
/* 去重 + 保持 CATEGORY_LIST 的固定顺序，报告顺序才稳定 */
|
||||
return CATEGORY_KEYS.filter(key => list.includes(key))
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
CATEGORY_LIST,
|
||||
CATEGORY_KEYS,
|
||||
TABLES,
|
||||
OWNER_COLUMNS,
|
||||
ORG_COLUMNS,
|
||||
tablesOfCategory,
|
||||
resolveCategories
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
/**
|
||||
* 业务数据导入内核。
|
||||
*
|
||||
* 三条硬规则（对应 Wayne 2026-10-23 定的口径）：
|
||||
* ① **归属一律给导入人** —— owner_id / created_by / assigned_to 全部改写成本次导入的账号；
|
||||
* ② **冲突跳过，绝不覆盖** —— 线上已有同一条记录就跳过，并在报告里点名；
|
||||
* ③ **父记录可补壳** —— 只导订单时，客户按公司名找不到就建一条「只有公司名」的客户，
|
||||
* 报告里单列「自动补建」，导完提示补全信息。
|
||||
*
|
||||
* 这个函数是「纯函数式」的：给它一个已打开的库实例，它只在这个实例里写。
|
||||
* 因此同一个实现既能跑在**真实库**（apply），也能跑在**库的副本**（preview）——
|
||||
* 预览报告与实际导入完全同源，不存在「预览说能导、真导却失败」的两套逻辑。
|
||||
*/
|
||||
const { TABLES, OWNER_COLUMNS, ORG_COLUMNS, resolveCategories, CATEGORY_LIST } = require('./catalog')
|
||||
const { tableInfo, foreignKeys, tableExists, decodeValue } = require('./bundle')
|
||||
|
||||
const quote = name => `"${String(name).replace(/"/g, '""')}"`
|
||||
const MAX_WARNINGS_PER_TABLE = 20
|
||||
|
||||
/** 导入目标组织：优先取导入人所属组织，否则退回第一个组织 */
|
||||
function targetOrgId(db, user) {
|
||||
if (user && user.org_id) return user.org_id
|
||||
const row = db.prepare('SELECT id FROM organizations ORDER BY id LIMIT 1').get()
|
||||
return row ? row.id : null
|
||||
}
|
||||
|
||||
function makeContext(db, user) {
|
||||
return {
|
||||
db,
|
||||
userId: user ? user.id : null,
|
||||
orgId: targetOrgId(db, user),
|
||||
now: new Date().toISOString(),
|
||||
ids: new Map(), // table -> Map(包内旧 id -> 本库新 id)
|
||||
stubTotals: new Map(),
|
||||
statements: new Map()
|
||||
}
|
||||
}
|
||||
|
||||
function idMap(ctx, table) {
|
||||
if (!ctx.ids.has(table)) ctx.ids.set(table, new Map())
|
||||
return ctx.ids.get(table)
|
||||
}
|
||||
|
||||
/**
|
||||
* 按自然键找已存在的记录（供去重与父记录解析共用）。
|
||||
*
|
||||
* 键里允许出现空值：product_groups 的键是「名称 + 品牌」，而品牌经常为空 ——
|
||||
* 若要求每列都非空才算可比较，这类表就会永远判定「不存在」，重复导入时成倍增长。
|
||||
* 所以空值按 `IS NULL` 比较，只要键里至少有一列有值就参与去重。
|
||||
*/
|
||||
function findByKey(db, table, keySpec, values) {
|
||||
const clauses = []
|
||||
const params = []
|
||||
for (const key of keySpec) {
|
||||
const value = values[key]
|
||||
if (value === undefined) return null
|
||||
if (value === null || value === '') {
|
||||
clauses.push(`(${quote(key)} IS NULL OR ${quote(key)} = '')`)
|
||||
} else {
|
||||
clauses.push(`${quote(key)} = ?`)
|
||||
params.push(value)
|
||||
}
|
||||
}
|
||||
if (!clauses.length) return null
|
||||
try {
|
||||
return db.prepare(`SELECT * FROM ${quote(table)} WHERE ${clauses.join(' AND ')} LIMIT 1`).get(...params) || null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** 父记录的自然键快照 → 列值对象 */
|
||||
function keyValuesFromSnapshot(parentSpec, snapshot) {
|
||||
if (snapshot === null || snapshot === undefined) return null
|
||||
if (typeof snapshot === 'object') return snapshot
|
||||
return { [parentSpec.key[0]]: snapshot }
|
||||
}
|
||||
/** 补一条最小父记录（客户只有公司名、供应商只有名字、产品组只有名字） */
|
||||
function createStub(ctx, parentSpec, keyValues) {
|
||||
const value = keyValues ? keyValues[parentSpec.key[0]] : null
|
||||
if (!value) return null
|
||||
const { db, now } = ctx
|
||||
if (parentSpec.table === 'customers') {
|
||||
/* 补出来的客户直接归导入人、状态置为名下（不给 pooled 状态，避免导完冒出一堆公海客户） */
|
||||
const result = db.prepare(`
|
||||
INSERT INTO customers (company, owner_id, created_by, org_id, status, pool_status, created_at)
|
||||
VALUES (?, ?, ?, ?, 'active', 'owned', ?)
|
||||
`).run(String(value), ctx.userId, ctx.userId, ctx.orgId, now)
|
||||
ctx.stubTotals.set('customers', (ctx.stubTotals.get('customers') || 0) + 1)
|
||||
return result.lastInsertRowid
|
||||
}
|
||||
if (parentSpec.table === 'suppliers') {
|
||||
const result = db.prepare('INSERT INTO suppliers (name, created_at) VALUES (?, ?)').run(String(value), now)
|
||||
ctx.stubTotals.set('suppliers', (ctx.stubTotals.get('suppliers') || 0) + 1)
|
||||
return result.lastInsertRowid
|
||||
}
|
||||
if (parentSpec.table === 'product_groups') {
|
||||
const result = db.prepare('INSERT INTO product_groups (name, created_at) VALUES (?, ?)').run(String(value), now)
|
||||
ctx.stubTotals.set('product_groups', (ctx.stubTotals.get('product_groups') || 0) + 1)
|
||||
return result.lastInsertRowid
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析一个外键值：
|
||||
* ① 本包里刚导进去的同一条（id 映射表命中）
|
||||
* ② 本库已存在（按父记录自然键查）
|
||||
* ③ 都没有 → 允许补壳的补一个，否则置空
|
||||
*/
|
||||
function resolveParent(ctx, ref, rawValue, snapshot) {
|
||||
const parentSpec = TABLES.find(item => item.table === ref.table)
|
||||
const mapped = ctx.ids.get(ref.table)
|
||||
if (mapped && mapped.has(rawValue)) return mapped.get(rawValue)
|
||||
|
||||
if (parentSpec && parentSpec.key && parentSpec.key.length) {
|
||||
/* 父表主键就是自然键（products.sku）：父记录在就沿用原值，不在就置空由约束决定成败 */
|
||||
if (ref.to && ref.to !== 'id') {
|
||||
const exists = ctx.db.prepare(`SELECT 1 FROM ${quote(ref.table)} WHERE ${quote(ref.to)} = ?`).get(rawValue)
|
||||
return exists ? rawValue : null
|
||||
}
|
||||
const keyValues = keyValuesFromSnapshot(parentSpec, snapshot)
|
||||
if (keyValues) {
|
||||
const row = findByKey(ctx.db, ref.table, parentSpec.key, keyValues)
|
||||
if (row) {
|
||||
if (mapped) mapped.set(rawValue, row.id)
|
||||
return row.id
|
||||
}
|
||||
if (parentSpec.stub) {
|
||||
const created = createStub(ctx, parentSpec, keyValues)
|
||||
if (created) {
|
||||
if (mapped) mapped.set(rawValue, created)
|
||||
return created
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function prepareStatement(ctx, sql) {
|
||||
if (!ctx.statements.has(sql)) ctx.statements.set(sql, ctx.db.prepare(sql))
|
||||
return ctx.statements.get(sql)
|
||||
}
|
||||
|
||||
/** 单表导入 */
|
||||
function importTable(ctx, spec, rows, info) {
|
||||
const stat = {
|
||||
table: spec.table,
|
||||
label: spec.label,
|
||||
category: spec.category,
|
||||
rows: rows.length,
|
||||
insert: 0,
|
||||
skip: 0,
|
||||
failed: 0,
|
||||
warnings: []
|
||||
}
|
||||
const warn = message => {
|
||||
if (stat.warnings.length < MAX_WARNINGS_PER_TABLE) stat.warnings.push(message)
|
||||
}
|
||||
|
||||
const columnsByName = new Map(info.columns.map(column => [column.name, column]))
|
||||
const fkByColumn = new Map(foreignKeys(ctx.db, spec.table).map(ref => [ref.from, ref]))
|
||||
const unknownColumns = new Set()
|
||||
|
||||
/* 去重用的列：普通表用目录里定义的业务键；流水表（price_history）用整行内容。
|
||||
整行比对时排除自增主键与「归属人 / 组织」—— 这两类本来就会被改写成本次导入的人，
|
||||
不该参与「是不是同一条记录」的判断。 */
|
||||
const keyColumns = spec.contentKey
|
||||
? info.columns
|
||||
.map(column => column.name)
|
||||
.filter(name => !(info.autoId && name === info.pk) && !OWNER_COLUMNS.has(name) && !ORG_COLUMNS.has(name))
|
||||
: spec.key
|
||||
|
||||
/* 流水表允许同一个包里出现逐列完全相同的两行（源库确实会这样：一次批量改价写两条）。
|
||||
它们对线上来说是两条独立的历史记录，必须都带过去；只跟**导入前**existing 的比对。
|
||||
普通表不允许：两个同货号的行塞进去会撞主键，那是数据本身有问题，跳过并报告更合适。 */
|
||||
const duplicatesInsidePackage = new Set()
|
||||
|
||||
for (let index = 0; index < rows.length; index++) {
|
||||
const raw = rows[index]
|
||||
try {
|
||||
const values = {}
|
||||
const refs = raw.__refs && typeof raw.__refs === 'object' ? raw.__refs : {}
|
||||
|
||||
for (const name of Object.keys(raw)) {
|
||||
if (name === '__refs') continue
|
||||
const column = columnsByName.get(name)
|
||||
if (!column) { unknownColumns.add(name); continue }
|
||||
const value = decodeValue(raw[name], info.blobs.has(name))
|
||||
|
||||
/* 归属人 / 归属组织：一律换成「本次导入的人 / 当前组织」 */
|
||||
if (OWNER_COLUMNS.has(name)) {
|
||||
values[name] = value === null ? null : ctx.userId
|
||||
continue
|
||||
}
|
||||
if (ORG_COLUMNS.has(name)) {
|
||||
values[name] = ctx.orgId
|
||||
continue
|
||||
}
|
||||
|
||||
const ref = fkByColumn.get(name)
|
||||
if (ref) {
|
||||
if (value === null) { values[name] = null; continue }
|
||||
const mapped = resolveParent(ctx, ref, value, refs[name])
|
||||
if (mapped === null || mapped === undefined) {
|
||||
/* 父记录既不在包里也补不出来：外键置空，由 NOT NULL 约束决定该行成败 */
|
||||
values[name] = null
|
||||
} else {
|
||||
values[name] = mapped
|
||||
}
|
||||
continue
|
||||
}
|
||||
values[name] = value
|
||||
}
|
||||
|
||||
/* 客户：归属强制给导入人；本来就是公海的客户改成「名下有主」，
|
||||
否则会出现「归我所有却标记在公海」这种自相矛盾的状态。 */
|
||||
if (spec.table === 'customers') {
|
||||
values.owner_id = ctx.userId
|
||||
values.created_by = ctx.userId
|
||||
values.org_id = ctx.orgId
|
||||
if (values.pool_status === 'public') {
|
||||
values.pool_status = 'owned'
|
||||
values.pool_at = null
|
||||
values.pool_note = null
|
||||
}
|
||||
}
|
||||
|
||||
/* 去重：键里至少一列有值才判定；某列为空按 NULL 比较，不因此放弃去重
|
||||
（否则「品牌为空的产品组」这种表每次重导都会翻倍） */
|
||||
const keyValues = {}
|
||||
let keyReady = false
|
||||
for (const key of keyColumns) {
|
||||
const value = values[key]
|
||||
if (value === undefined || value === null || value === '') {
|
||||
keyValues[key] = null
|
||||
continue
|
||||
}
|
||||
keyValues[key] = value
|
||||
keyReady = true
|
||||
}
|
||||
if (keyReady) {
|
||||
const fingerprint = keyColumns.map(key => `${key}=${keyValues[key] === null ? '\u0000' : keyValues[key]}`).join('\u0001')
|
||||
const repeatInsidePackage = duplicatesInsidePackage.has(fingerprint)
|
||||
const existing = (!repeatInsidePackage || !spec.contentKey)
|
||||
? findByKey(ctx.db, spec.table, keyColumns, keyValues)
|
||||
: null
|
||||
if (existing) {
|
||||
stat.skip++
|
||||
if (info.autoId && raw.id !== undefined && raw.id !== null && existing[info.pk]) {
|
||||
idMap(ctx, spec.table).set(raw.id, existing[info.pk])
|
||||
}
|
||||
continue
|
||||
}
|
||||
duplicatesInsidePackage.add(fingerprint)
|
||||
}
|
||||
|
||||
const names = Object.keys(values).filter(name => !(info.autoId && name === info.pk))
|
||||
if (!names.length) throw new Error('没有可写入的字段')
|
||||
|
||||
const sql = `INSERT INTO ${quote(spec.table)} (${names.map(quote).join(', ')}) VALUES (${names.map(() => '?').join(', ')})`
|
||||
const result = prepareStatement(ctx, sql).run(...names.map(name => values[name]))
|
||||
|
||||
stat.insert++
|
||||
if (info.autoId) {
|
||||
const oldId = raw.id
|
||||
if (oldId !== undefined && oldId !== null) idMap(ctx, spec.table).set(oldId, result.lastInsertRowid)
|
||||
}
|
||||
} catch (error) {
|
||||
stat.failed++
|
||||
warn(`第 ${index + 1} 行：${String(error.message || error).slice(0, 160)}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (unknownColumns.size) {
|
||||
warn(`目标库缺少列 ${[...unknownColumns].join('、')}（版本较旧？），这些字段已忽略`)
|
||||
}
|
||||
return stat
|
||||
}
|
||||
|
||||
/**
|
||||
* 把数据包写进给定的库实例。
|
||||
* 调用方负责包在事务里（见 index.js 的 applyImport）——这里出现异常就整体回滚。
|
||||
*/
|
||||
function importInto(db, bundle, { categories, user }) {
|
||||
const selected = new Set(resolveCategories(categories))
|
||||
const ctx = makeContext(db, user)
|
||||
const tables = []
|
||||
const missing = []
|
||||
|
||||
for (const spec of TABLES) {
|
||||
if (!selected.has(spec.category)) continue
|
||||
const rows = bundle.rows.get(spec.table)
|
||||
if (!rows || !rows.length) continue
|
||||
if (!tableExists(db, spec.table)) { missing.push(spec.table); continue }
|
||||
const info = tableInfo(db, spec.table)
|
||||
if (!info) { missing.push(spec.table); continue }
|
||||
tables.push(importTable(ctx, spec, rows, info))
|
||||
}
|
||||
|
||||
const totals = tables.reduce((acc, item) => {
|
||||
acc.rows += item.rows
|
||||
acc.insert += item.insert
|
||||
acc.skip += item.skip
|
||||
acc.failed += item.failed
|
||||
return acc
|
||||
}, { rows: 0, insert: 0, skip: 0, failed: 0 })
|
||||
|
||||
const stubbed = [...ctx.stubTotals.entries()].map(([table, count]) => ({
|
||||
table,
|
||||
label: (TABLES.find(item => item.table === table) || {}).label || table,
|
||||
count
|
||||
}))
|
||||
|
||||
return {
|
||||
categories: CATEGORY_LIST.filter(item => selected.has(item.key)).map(item => ({ key: item.key, label: item.label })),
|
||||
tables,
|
||||
totals,
|
||||
stubbed,
|
||||
missing
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { importInto, targetOrgId }
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* 数据迁移的门面：把「读包、预览、真导、备份」串起来，路由只调这一层。
|
||||
*
|
||||
* 预览为什么要开一个库副本：
|
||||
* 预览必须回答「照这样导，线上会变成什么样」。如果只是把行数点一遍，就答不了
|
||||
* 「哪一行会因为外键或非空约束失败」。所以预览在做一次 VACUUM INTO 出来的副本上
|
||||
* **真跑一遍导入**，跑完把副本删掉 —— 报告里的数字与实际导入完全同源。
|
||||
* 库只有几 MB，拷一份比维护两套判断逻辑便宜得多。
|
||||
*/
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const crypto = require('crypto')
|
||||
const Database = require('better-sqlite3-multiple-ciphers')
|
||||
|
||||
const { CATEGORY_LIST, TABLES, resolveCategories } = require('./catalog')
|
||||
const { writeBundle, readBundle, tableExists } = require('./bundle')
|
||||
const { importInto } = require('./import')
|
||||
|
||||
const escapePath = file => String(file).replace(/'/g, "''")
|
||||
|
||||
/** 本机各类别各表当前行数（界面用来显示「这次会导出多少」） */
|
||||
function listCatalog(db) {
|
||||
return CATEGORY_LIST.map(category => ({
|
||||
key: category.key,
|
||||
label: category.label,
|
||||
description: category.description,
|
||||
tables: TABLES.filter(spec => spec.category === category.key).map(spec => {
|
||||
let rows = 0
|
||||
if (tableExists(db, spec.table)) {
|
||||
try { rows = db.prepare(`SELECT COUNT(*) AS n FROM "${spec.table}"`).get().n } catch { rows = 0 }
|
||||
}
|
||||
return { table: spec.table, label: spec.label, rows }
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/** 导出到可写流（HTTP 响应直接接） */
|
||||
async function exportToStream(db, output, options) {
|
||||
await writeBundle(db, output, options)
|
||||
}
|
||||
|
||||
/** 导出的文件名：带时间戳，便于在服务器上分辨先后 */
|
||||
function exportFileName(date = new Date()) {
|
||||
const pad = value => String(value).padStart(2, '0')
|
||||
const stamp = `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}-${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`
|
||||
return `happy-export-${stamp}.hdata.gz`
|
||||
}
|
||||
|
||||
function readFileMeta(bundle, filePath, fileName) {
|
||||
return {
|
||||
name: fileName || path.basename(filePath),
|
||||
size: bundle.size,
|
||||
exportedAt: bundle.header.exportedAt,
|
||||
exporter: bundle.header.exporter || null,
|
||||
app: bundle.header.app,
|
||||
version: bundle.header.version,
|
||||
categories: bundle.categories
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 预览：在库副本上真跑一遍导入，返回与实际导入同源的报告。
|
||||
* 副本用完即删，线上库不受任何影响。
|
||||
*/
|
||||
async function previewImport(sourceDb, filePath, { categories, user, workDir, fileName }) {
|
||||
fs.mkdirSync(workDir, { recursive: true })
|
||||
const bundle = await readBundle(filePath)
|
||||
const selected = resolveCategories(categories && categories.length ? categories : bundle.categories)
|
||||
const available = selected.filter(key => bundle.categories.includes(key))
|
||||
if (!available.length) {
|
||||
const error = new Error('所选类别在数据包里没有内容，无法导入')
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
|
||||
const copyPath = path.join(workDir, `preview-${crypto.randomBytes(6).toString('hex')}.db`)
|
||||
if (fs.existsSync(copyPath)) fs.unlinkSync(copyPath)
|
||||
sourceDb.exec(`VACUUM INTO '${escapePath(copyPath)}'`)
|
||||
|
||||
const copy = new Database(copyPath)
|
||||
try {
|
||||
copy.pragma('foreign_keys = ON')
|
||||
copy.pragma('busy_timeout = 5000')
|
||||
const plan = copy.transaction(() => importInto(copy, bundle, { categories: available, user }))()
|
||||
return { file: readFileMeta(bundle, filePath, fileName), plan, availableCategories: available }
|
||||
} finally {
|
||||
try { copy.close() } catch { /* 副本关闭失败不影响结果 */ }
|
||||
try { fs.unlinkSync(copyPath) } catch { /* 副本已被占用就留着，下次预览会覆盖 */ }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 真正导入：先给线上库做一份完整备份（留档可回滚），再在一个事务里写入。
|
||||
* 任何未预期的异常都会整体回滚，备份文件保留在 backupDir。
|
||||
*/
|
||||
async function applyImport(db, filePath, { categories, user, backupDir, fileName }) {
|
||||
fs.mkdirSync(backupDir, { recursive: true })
|
||||
const bundle = await readBundle(filePath)
|
||||
const selected = resolveCategories(categories && categories.length ? categories : bundle.categories)
|
||||
const available = selected.filter(key => bundle.categories.includes(key))
|
||||
if (!available.length) {
|
||||
const error = new Error('所选类别在数据包里没有内容，无法导入')
|
||||
error.statusCode = 400
|
||||
throw error
|
||||
}
|
||||
|
||||
/* 时间戳精确到秒，再拼一段随机后缀：同一秒内连续导两次也不会撞出
|
||||
「output file already exists」，而文件名照样按时间可排序。 */
|
||||
const stamp = new Date().toISOString().replace(/[-:TZ.]/g, '').slice(0, 14)
|
||||
const backupPath = path.join(backupDir, `pre-import-${stamp}-${crypto.randomBytes(3).toString('hex')}.db`)
|
||||
db.exec(`VACUUM INTO '${escapePath(backupPath)}'`)
|
||||
|
||||
const plan = db.transaction(() => importInto(db, bundle, { categories: available, user }))()
|
||||
return { file: readFileMeta(bundle, filePath, fileName), plan, backup: backupPath, availableCategories: available }
|
||||
}
|
||||
|
||||
/** 清理过期的上传包与预览副本（默认 24 小时） */
|
||||
function cleanupUploads(dir, { maxAgeMs = 24 * 60 * 60 * 1000 } = {}) {
|
||||
let removed = 0
|
||||
try {
|
||||
for (const name of fs.readdirSync(dir)) {
|
||||
const file = path.join(dir, name)
|
||||
try {
|
||||
const stat = fs.statSync(file)
|
||||
if (Date.now() - stat.mtimeMs > maxAgeMs) { fs.unlinkSync(file); removed++ }
|
||||
} catch { /* 单个文件失败不影响其它清理 */ }
|
||||
}
|
||||
} catch { /* 目录还不存在 */ }
|
||||
return removed
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
listCatalog,
|
||||
exportToStream,
|
||||
exportFileName,
|
||||
previewImport,
|
||||
applyImport,
|
||||
cleanupUploads
|
||||
}
|
||||
Reference in New Issue
Block a user