기존 평면 이미지 자산 샤딩 마이그레이션 추가
This commit is contained in:
@@ -8,6 +8,7 @@
|
||||
"start": "APP_ORIGIN=http://localhost:5173 DB_HOST=127.0.0.1 DB_PORT=3307 DB_USER=tier_cursor DB_PASSWORD=tier_cursor1234 DB_NAME=tier_cursor node index.js",
|
||||
"images:backfill": "DB_HOST=127.0.0.1 DB_PORT=3307 DB_USER=tier_cursor DB_PASSWORD=tier_cursor1234 DB_NAME=tier_cursor node scripts/backfill-legacy-image-assets.js",
|
||||
"images:migrate-legacy": "DB_HOST=127.0.0.1 DB_PORT=3307 DB_USER=tier_cursor DB_PASSWORD=tier_cursor1234 DB_NAME=tier_cursor node scripts/migrate-legacy-uploads-to-assets.js",
|
||||
"images:shard-assets": "DB_HOST=127.0.0.1 DB_PORT=3307 DB_USER=tier_cursor DB_PASSWORD=tier_cursor1234 DB_NAME=tier_cursor node scripts/migrate-flat-assets-to-sharded.js",
|
||||
"uploads:cleanup-legacy": "DB_HOST=127.0.0.1 DB_PORT=3307 DB_USER=tier_cursor DB_PASSWORD=tier_cursor1234 DB_NAME=tier_cursor node scripts/cleanup-unreferenced-legacy-uploads.js"
|
||||
},
|
||||
"keywords": [],
|
||||
|
||||
102
backend/scripts/migrate-flat-assets-to-sharded.js
Normal file
102
backend/scripts/migrate-flat-assets-to-sharded.js
Normal file
@@ -0,0 +1,102 @@
|
||||
const fs = require('fs/promises')
|
||||
const path = require('path')
|
||||
const {
|
||||
ensureData,
|
||||
closePool,
|
||||
updateImageAssetSrc,
|
||||
replaceUploadSourceReferences,
|
||||
} = require('../src/db')
|
||||
|
||||
const BACKEND_ROOT = path.join(__dirname, '..')
|
||||
const ASSETS_ROOT = path.join(BACKEND_ROOT, 'uploads', 'assets')
|
||||
const FLAT_ASSET_PATTERN = /^\/uploads\/assets\/[^/]+$/
|
||||
|
||||
function getShardedAssetSrc(src) {
|
||||
const filename = path.basename(src || '')
|
||||
const shardDirectory = filename.slice(0, 2)
|
||||
if (!filename || shardDirectory.length < 2) return ''
|
||||
return `/uploads/assets/${shardDirectory}/${filename}`
|
||||
}
|
||||
|
||||
async function moveAssetFile(fromSrc, toSrc) {
|
||||
const fromPath = path.join(BACKEND_ROOT, fromSrc.replace(/^\//, ''))
|
||||
const toPath = path.join(BACKEND_ROOT, toSrc.replace(/^\//, ''))
|
||||
await fs.mkdir(path.dirname(toPath), { recursive: true })
|
||||
|
||||
try {
|
||||
await fs.rename(fromPath, toPath)
|
||||
return 'moved'
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.access(toPath)
|
||||
return 'already_moved'
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return 'missing'
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await ensureData()
|
||||
|
||||
let dirEntries = []
|
||||
try {
|
||||
dirEntries = await fs.readdir(ASSETS_ROOT, { withFileTypes: true })
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
}
|
||||
|
||||
const flatAssets = dirEntries
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => ({ src: `/uploads/assets/${entry.name}` }))
|
||||
.filter((asset) => FLAT_ASSET_PATTERN.test(asset.src || ''))
|
||||
const summary = {
|
||||
scanned: flatAssets.length,
|
||||
migrated: 0,
|
||||
alreadyMoved: 0,
|
||||
skipped: 0,
|
||||
missingFiles: 0,
|
||||
failed: 0,
|
||||
updatedRows: 0,
|
||||
}
|
||||
|
||||
for (const asset of flatAssets) {
|
||||
const nextSrc = getShardedAssetSrc(asset.src)
|
||||
if (!nextSrc) {
|
||||
summary.skipped += 1
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
const moveStatus = await moveAssetFile(asset.src, nextSrc)
|
||||
if (moveStatus === 'missing') {
|
||||
summary.missingFiles += 1
|
||||
continue
|
||||
}
|
||||
|
||||
await updateImageAssetSrc({ fromSrc: asset.src, toSrc: nextSrc })
|
||||
const replaced = await replaceUploadSourceReferences({ fromSrc: asset.src, toSrc: nextSrc })
|
||||
summary.updatedRows += Number(replaced.updatedRows || 0)
|
||||
|
||||
if (moveStatus === 'already_moved') summary.alreadyMoved += 1
|
||||
else summary.migrated += 1
|
||||
} catch (error) {
|
||||
summary.failed += 1
|
||||
console.error('[migrate-flat-assets-to-sharded] failed:', asset.src, error?.message || error)
|
||||
}
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(summary, null, 2))
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((error) => {
|
||||
console.error(error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
.finally(async () => {
|
||||
await closePool()
|
||||
})
|
||||
@@ -1300,6 +1300,12 @@ async function findImageAssetById(id) {
|
||||
return mapImageAssetRow(rows[0])
|
||||
}
|
||||
|
||||
async function updateImageAssetSrc({ fromSrc, toSrc }) {
|
||||
if (!fromSrc || !toSrc || fromSrc === toSrc) return null
|
||||
await query('UPDATE image_assets SET src = ? WHERE src = ?', [toSrc, fromSrc])
|
||||
return findImageAssetBySrc(toSrc)
|
||||
}
|
||||
|
||||
async function getReferencedUploadFootprint() {
|
||||
const [referencedSrcs, assets] = await Promise.all([listReferencedUploadSources(), listImageAssets()])
|
||||
const assetMap = new Map(assets.map((asset) => [asset.src, asset]))
|
||||
@@ -3087,6 +3093,7 @@ module.exports = {
|
||||
findImageAssetByHash,
|
||||
findImageAssetBySrc,
|
||||
findImageAssetById,
|
||||
updateImageAssetSrc,
|
||||
createImageAsset,
|
||||
createImageOptimizationJob,
|
||||
findImageOptimizationJobById,
|
||||
@@ -3094,6 +3101,7 @@ module.exports = {
|
||||
listRecentImageOptimizationJobs,
|
||||
listUnusedImageAssets,
|
||||
deleteImageAssets,
|
||||
listImageAssets,
|
||||
listReferencedUploadSources,
|
||||
listReferencedUploadUsage,
|
||||
replaceUploadSourceReferences,
|
||||
|
||||
Reference in New Issue
Block a user