zenn/sori.studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

게시물 Export 용량 기준 분할 추가 v1.5.25

Browse Source
This commit is contained in:
zenn
2026-06-01 16:20:35 +09:00
parent 5735fd5046
commit 212bd3f34f
14 changed files with 316 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
server/repositories/post-export-repository.js
View File

@@ -20,9 +20,12 @@ const EXPORT_FILE_STATUS = {
FAILED: 'failed'
}
const DEFAULT_CHUNK_SIZE = 100
const DEFAULT_CHUNK_SIZE = 500
const MAX_CHUNK_SIZE = 500
const DEFAULT_RETENTION_DAYS = 100
const DEFAULT_MAX_FILE_SIZE_BYTES = 500 * 1024 * 1024
const MIN_MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024
const MAX_MAX_FILE_SIZE_BYTES = 2 * 1024 * 1024 * 1024
const UPLOAD_ROOT = join(process.cwd(), 'public', 'uploads')
const EXPORT_ROOT_NAME = 'exports'
const runningPostExportJobIds = new Set()
@@ -162,6 +165,13 @@ const collectLocalUploadUrls = (content) => {
return [...new Set(matches.filter(isLocalUploadUrl))]
}
/**
* UTF-8 바이트 길이를 계산한다.
* @param {unknown} value - 원본 값
* @returns {number} 바이트 길이
*/
const getUtf8ByteLength = (value) => Buffer.byteLength(String(value || ''), 'utf8')
/**
* 게시물 frontmatter를 만든다.
* @param {Object} post - 게시물
@@ -296,6 +306,7 @@ const mapPostExportJobRow = (row) => ({
processedCount: Number(row.processed_count || 0),
currentPartIndex: row.current_part_index ? Number(row.current_part_index) : null,
chunkSize: Number(row.chunk_size || DEFAULT_CHUNK_SIZE),
maxFileSizeBytes: Number(row.max_file_size_bytes || DEFAULT_MAX_FILE_SIZE_BYTES),
retentionDays: Number(row.retention_days || DEFAULT_RETENTION_DAYS),
dateFrom: row.date_from ? row.date_from.toISOString() : null,
dateTo: row.date_to ? row.date_to.toISOString() : null,
@@ -303,6 +314,7 @@ const mapPostExportJobRow = (row) => ({
expiresAt: row.expires_at ? row.expires_at.toISOString() : null,
message: row.message || '',
progressMessage: row.progress_message || '',
errorDetail: row.error_detail || '',
createdAt: row.created_at.toISOString(),
updatedAt: row.updated_at.toISOString(),
startedAt: row.started_at ? row.started_at.toISOString() : null,
@@ -360,6 +372,21 @@ const normalizeRetentionDays = (value) => {
return Math.min(Math.max(Math.trunc(parsed), 1), DEFAULT_RETENTION_DAYS)
}
/**
* Export 분할 ZIP 최대 용량을 안전 범위로 보정한다.
* @param {unknown} value - 입력 바이트
* @returns {number} 보정된 최대 용량
*/
const normalizeMaxFileSizeBytes = (value) => {
const parsed = Number(value || getRuntimeEnvValue('POST_EXPORT_MAX_FILE_SIZE_BYTES', 'postExportMaxFileSizeBytes', DEFAULT_MAX_FILE_SIZE_BYTES))
if (!Number.isFinite(parsed)) {
return DEFAULT_MAX_FILE_SIZE_BYTES
}
return Math.min(Math.max(Math.trunc(parsed), MIN_MAX_FILE_SIZE_BYTES), MAX_MAX_FILE_SIZE_BYTES)
}
/**
* 날짜 입력을 안전하게 보정한다.
* @param {unknown} value - 날짜 입력
@@ -477,6 +504,161 @@ const getExportSiteName = async (sql) => {
return rows[0]?.title || getDefaultSiteSettings().title
}
/**
* 게시물에 포함된 내부 업로드 파일 크기를 추산한다.
* @param {Object} post - 게시물
* @returns {Promise<number>} 내부 자산 바이트 합계
*/
const estimatePostAssetBytes = async (post) => {
const urls = new Set(collectLocalUploadUrls(post.content))
if (isLocalUploadUrl(post.featured_image)) {
urls.add(post.featured_image)
}
if (isLocalUploadUrl(post.og_image)) {
urls.add(post.og_image)
}
let total = 0
for (const url of urls) {
const diskPath = resolveUploadUrlToDiskPath(url)
if (!diskPath) {
continue
}
try {
const fileStat = await stat(diskPath)
if (fileStat.isFile()) {
total += fileStat.size
}
} catch {}
}
return total
}
/**
* 게시물 하나가 ZIP 안에서 차지할 대략적인 크기를 추산한다.
* @param {Object} post - 게시물
* @returns {Promise<number>} 추산 바이트
*/
const estimatePostExportBytes = async (post) => {
const frontmatterReserve = 4096
const zipEntryReserve = 2048
const contentBytes = getUtf8ByteLength(post.content)
const metaBytes = getUtf8ByteLength([
post.title,
post.slug,
post.excerpt,
post.seo_title,
post.seo_description,
post.canonical_url
].join('\n'))
const assetBytes = await estimatePostAssetBytes(post)
return Math.max(frontmatterReserve + zipEntryReserve + contentBytes + metaBytes + assetBytes, 1)
}
/**
* Export 계획 대상 게시물을 순서대로 조회한다.
* @param {ReturnType<import('postgres')>} sql - PostgreSQL 클라이언트
* @param {Object} input - 조회 입력
* @returns {Promise<Array>} 계획 대상 게시물 목록
*/
const listPostsForExportPlanning = async (sql, input) => {
const dateCondition = createPostExportDateCondition(sql, input.exportDateRange)
if (input.scope === 'author') {
return sql`
SELECT
id,
title,
slug,
content,
excerpt,
seo_title,
seo_description,
canonical_url,
featured_image,
og_image,
COALESCE(published_at, created_at) AS export_date
FROM posts
WHERE author_id = ${input.requestedBy}
AND ${dateCondition}
ORDER BY COALESCE(published_at, created_at) ASC, id ASC
`
}
return sql`
SELECT
id,
title,
slug,
content,
excerpt,
seo_title,
seo_description,
canonical_url,
featured_image,
og_image,
COALESCE(published_at, created_at) AS export_date
FROM posts
WHERE ${dateCondition}
ORDER BY COALESCE(published_at, created_at) ASC, id ASC
`
}
/**
* 게시물 수와 추산 용량을 함께 사용해 Export 분할 계획을 만든다.
* @param {Array<Object>} posts - 순서가 확정된 게시물 목록
* @param {Object} input - 계획 옵션
* @returns {Promise<Array<{ partIndex: number, postStart: number, postEnd: number }>>} 분할 계획
*/
const createPostExportFilePlan = async (posts, input) => {
const plan = []
let currentStart = 1
let currentEnd = 0
let currentCount = 0
let currentBytes = 0
for (let index = 0; index < posts.length; index += 1) {
const postNumber = index + 1
const postBytes = await estimatePostExportBytes(posts[index])
const shouldStartNext = currentCount > 0 && (
currentCount >= input.chunkSize
|| currentBytes + postBytes > input.maxFileSizeBytes
)
if (shouldStartNext) {
plan.push({
partIndex: plan.length + 1,
postStart: currentStart,
postEnd: currentEnd
})
currentStart = postNumber
currentCount = 0
currentBytes = 0
}
currentEnd = postNumber
currentCount += 1
currentBytes += postBytes
}
if (currentCount > 0) {
plan.push({
partIndex: plan.length + 1,
postStart: currentStart,
postEnd: currentEnd
})
}
return plan
}
/**
* Export 완료 안내 이메일을 보낸다.
* @param {Object} job - Export 작업
@@ -639,6 +821,7 @@ export const listQueuedPostExportJobIds = async () => {
* @param {Date|string|null} [input.dateTo] - 종료일
* @param {string} [input.rangeLabel] - 범위 라벨
* @param {number} [input.chunkSize] - 분할당 게시물 수
* @param {number} [input.maxFileSizeBytes] - 분할 ZIP 최대 추산 용량
* @param {number} [input.retentionDays] - 산출물 보존 일수
* @returns {Promise<Object>} 생성된 export 작업
*/
@@ -653,6 +836,7 @@ export const createPostExportJob = async (input) => {
const scope = input.scope === 'author' ? 'author' : 'all'
const chunkSize = normalizeChunkSize(input.chunkSize)
const maxFileSizeBytes = normalizeMaxFileSizeBytes(input.maxFileSizeBytes)
const retentionDays = normalizeRetentionDays(input.retentionDays)
const exportDateRange = normalizeExportDateRange(input)
const siteName = sanitizeFilenameSegment(await getExportSiteName(sql))
@@ -660,20 +844,18 @@ export const createPostExportJob = async (input) => {
const expiresAt = new Date(Date.now() + retentionDays * 24 * 60 * 60 * 1000)
const [createdJob] = await sql.begin(async (transaction) => {
const dateCondition = createPostExportDateCondition(transaction, exportDateRange)
const [{ count }] = scope === 'author'
? await transaction`
SELECT COUNT(*)::int AS count
FROM posts
WHERE author_id = ${input.requestedBy}
AND ${dateCondition}
`
: await transaction`
SELECT COUNT(*)::int AS count
FROM posts
WHERE ${dateCondition}
`
const postCount = Number(count || 0)
const planningPosts = await listPostsForExportPlanning(transaction, {
scope,
requestedBy: input.requestedBy,
exportDateRange
})
const postCount = planningPosts.length
const filePlan = postCount > 0
? await createPostExportFilePlan(planningPosts, {
chunkSize,
maxFileSizeBytes
})
: []
const status = postCount > 0 ? EXPORT_STATUS.QUEUED : EXPORT_STATUS.READY
const message = postCount > 0
? 'Export 작업이 대기열에 등록되었습니다.'
@@ -686,6 +868,7 @@ export const createPostExportJob = async (input) => {
scope,
post_count,
chunk_size,
max_file_size_bytes,
retention_days,
date_from,
date_to,
@@ -701,6 +884,7 @@ export const createPostExportJob = async (input) => {
${scope},
${postCount},
${chunkSize},
${maxFileSizeBytes},
${retentionDays},
${exportDateRange.dateFrom},
${exportDateRange.dateTo},
@@ -714,20 +898,14 @@ export const createPostExportJob = async (input) => {
const job = jobRows[0]
if (postCount > 0) {
const partCount = Math.ceil(postCount / chunkSize)
const fileInputs = Array.from({ length: partCount }, (_, index) => {
const postStart = index * chunkSize + 1
const postEnd = Math.min((index + 1) * chunkSize, postCount)
return {
jobId: job.id,
partIndex: index + 1,
postStart,
postEnd,
fileName: `${siteName}_${fileRangeName}_${postStart}-${postEnd}.zip`,
status: EXPORT_FILE_STATUS.PENDING
}
})
const fileInputs = filePlan.map((part) => ({
jobId: job.id,
partIndex: part.partIndex,
postStart: part.postStart,
postEnd: part.postEnd,
fileName: `${siteName}_${fileRangeName}_${part.postStart}-${part.postEnd}.zip`,
status: EXPORT_FILE_STATUS.PENDING
}))
for (const fileInput of fileInputs) {
await transaction`
@@ -808,6 +986,7 @@ const markPostExportJobProcessing = async (sql, jobId) => {
SET status = ${EXPORT_STATUS.PROCESSING},
started_at = COALESCE(started_at, now()),
progress_message = 'Export 파일 생성을 시작했습니다.',
error_detail = '',
updated_at = now()
WHERE id = ${jobId}
AND status IN (${EXPORT_STATUS.QUEUED}, ${EXPORT_STATUS.PROCESSING})
@@ -846,6 +1025,7 @@ const markPostExportJobReady = async (sql, jobId, postCount) => {
current_part_index = null,
message = '게시물 Export 파일 생성이 완료되었습니다.',
progress_message = '생성이 완료되었습니다.',
error_detail = '',
completed_at = now(),
updated_at = now()
WHERE id = ${jobId}
@@ -861,12 +1041,14 @@ const markPostExportJobReady = async (sql, jobId, postCount) => {
*/
const markPostExportJobFailed = async (sql, jobId, error) => {
const message = error instanceof Error ? error.message : '알 수 없는 오류'
const stack = error instanceof Error && error.stack ? error.stack : message
await sql`
UPDATE post_export_jobs
SET status = ${EXPORT_STATUS.FAILED},
message = ${`게시물 Export 생성 실패: ${message}`},
progress_message = ${message},
error_detail = ${stack.slice(0, 8000)},
current_part_index = null,
completed_at = now(),
updated_at = now()
@@ -1144,6 +1326,7 @@ export const retryPostExportJob = async (jobId) => {
SET status = ${EXPORT_STATUS.QUEUED},
message = '실패한 Export 작업을 다시 대기열에 등록했습니다.',
progress_message = '준비 완료 파일은 유지하고 실패 지점부터 다시 생성합니다.',
error_detail = '',
current_part_index = null,
completed_at = null,
updated_at = now()

2
server/routes/admin/api/posts/export-jobs.post.js
View File

@@ -13,6 +13,7 @@ const postExportJobInputSchema = z.object({
dateFrom: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(),
dateTo: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(),
chunkSize: z.number().int().min(1).max(500).optional(),
maxFileSizeBytes: z.number().int().min(10485760).max(2147483648).optional(),
retentionDays: z.number().int().min(1).max(100).optional()
}).default({})
@@ -120,6 +121,7 @@ export default defineEventHandler(async (event) => {
dateTo: dateRange.dateTo,
rangeLabel: dateRange.rangeLabel,
chunkSize: input.chunkSize,
maxFileSizeBytes: input.maxFileSizeBytes,
retentionDays: input.retentionDays
})