Initial commit

This commit is contained in:
admin
2026-05-26 22:46:00 +02:00
commit 7e2c2ff89c
256 changed files with 51523 additions and 0 deletions
+126
View File
@@ -0,0 +1,126 @@
import "server-only";
import { NextRequest, NextResponse } from "next/server";
const LOCAL_HOSTS = new Set(["127.0.0.1", "::1", "localhost"]);
function bareHost(value: string | null): string {
if (!value) return "";
const trimmed = value.trim().toLowerCase();
if (trimmed === "::1") return trimmed;
if (trimmed.startsWith("[") && trimmed.includes("]")) {
return trimmed.slice(1, trimmed.indexOf("]"));
}
return trimmed.split(":")[0];
}
function isLocalHost(value: string | null): boolean {
return LOCAL_HOSTS.has(bareHost(value));
}
function trustedLanEnabled(): boolean {
return process.env.PINKUDEX_TRUSTED_LAN === "1";
}
function trustedHostnames(): Set<string> {
const raw = process.env.PINKUDEX_TRUSTED_HOSTNAMES;
if (!raw) return new Set();
return new Set(
raw
.split(",")
.map((v) => v.trim().toLowerCase())
.filter(Boolean),
);
}
function parseIPv4(value: string): number[] | null {
const parts = value.split(".");
if (parts.length !== 4) return null;
const octets: number[] = [];
for (const p of parts) {
if (!/^\d{1,3}$/.test(p)) return null;
const n = Number(p);
if (n < 0 || n > 255) return null;
octets.push(n);
}
return octets;
}
function isPrivateIPv4(value: string): boolean {
const o = parseIPv4(value);
if (!o) return false;
// 127.0.0.0/8 (loopback)
if (o[0] === 127) return true;
// 10.0.0.0/8
if (o[0] === 10) return true;
// 172.16.0.0/12
if (o[0] === 172 && o[1] >= 16 && o[1] <= 31) return true;
// 192.168.0.0/16
if (o[0] === 192 && o[1] === 168) return true;
// 100.64.0.0/10 (CGNAT / Tailscale)
if (o[0] === 100 && o[1] >= 64 && o[1] <= 127) return true;
return false;
}
function isPrivateIPv6(value: string): boolean {
const v = value.toLowerCase();
if (v === "::1") return true;
// fc00::/7 (ULA): first byte 0xfc or 0xfd
if (v.startsWith("fc") || v.startsWith("fd")) return true;
// fe80::/10 (link-local)
if (v.startsWith("fe8") || v.startsWith("fe9") || v.startsWith("fea") || v.startsWith("feb")) return true;
return false;
}
function isTrustedHost(value: string | null): boolean {
const host = bareHost(value);
if (!host) return false;
if (LOCAL_HOSTS.has(host)) return true;
if (!trustedLanEnabled()) return false;
if (host.includes(":") || /^[0-9a-f:]+$/i.test(host)) {
if (isPrivateIPv6(host)) return true;
}
if (isPrivateIPv4(host)) return true;
if (trustedHostnames().has(host)) return true;
return false;
}
function forwardedFor(req: NextRequest): string[] {
const out: string[] = [];
const xff = req.headers.get("x-forwarded-for");
if (xff) out.push(...xff.split(",").map((v) => v.trim()).filter(Boolean));
const realIp = req.headers.get("x-real-ip");
if (realIp) out.push(realIp.trim());
const forwarded = req.headers.get("forwarded");
if (forwarded) {
for (const part of forwarded.split(",")) {
const m = part.match(/(?:^|;)\s*for="?(\[?[^\]";,]+]?)"?/i);
if (m) out.push(m[1].trim());
}
}
return out;
}
function sameOriginHeaderIsTrusted(req: NextRequest, header: "origin" | "referer"): boolean {
const raw = req.headers.get(header);
if (!raw) return true;
try {
return isTrustedHost(new URL(raw).hostname);
} catch {
return false;
}
}
export function assertLocalRequest(req: NextRequest): NextResponse | null {
const hostIsLocal = isTrustedHost(req.headers.get("host")) && isTrustedHost(req.nextUrl.hostname);
const forwarded = forwardedFor(req);
const forwardedIsLocal = forwarded.length === 0 || forwarded.every(isTrustedHost);
const originIsLocal =
sameOriginHeaderIsTrusted(req, "origin") && sameOriginHeaderIsTrusted(req, "referer");
if (hostIsLocal && forwardedIsLocal && originIsLocal) return null;
return NextResponse.json(
{ error: "This endpoint is only available from the local machine." },
{ status: 403 },
);
}
export { isLocalHost };
Binary file not shown.
+43
View File
@@ -0,0 +1,43 @@
/**
* URL builders for our asset routes. The path basename is the friendly filename
* (used as the browser tab title and "Save as…" suggestion). The query string
* carries the actual lookup key.
*/
function safe(name: string): string {
// Browser-safe: encode anything that's not alphanumeric, dash, dot, or underscore.
return encodeURIComponent(name);
}
export function imageUrl(opts: { id: number; code: string | null; ext?: string; v?: string }): string {
const ext = opts.ext ?? ".jpg";
const params = new URLSearchParams({ id: String(opts.id) });
if (opts.v) params.set("v", opts.v);
if (opts.code) {
const friendly = `${opts.code}${ext}`;
return `/api/image/${safe(friendly)}?${params.toString()}`;
}
const friendly = `image-${opts.id}${ext}`;
return `/api/image/${safe(friendly)}?${params.toString()}`;
}
export function thumbUrl(opts: { thumbPath: string; code?: string | null; id?: number | null }): string {
// thumbPath is the SHA-based filename ("<sha>.webp"). Friendly name uses code if available.
const friendly = `${opts.code ?? (opts.id != null ? `image-${opts.id}` : "thumb")}.webp`;
return `/api/thumb/${safe(friendly)}?p=${encodeURIComponent(opts.thumbPath)}`;
}
export function portraitUrl(opts: { path: string; slug?: string | null; slot?: "1" | "2" | "3" | "4" | "h" }): string {
const ext = opts.path.match(/\.[^.]+$/)?.[0] ?? ".jpg";
const slotSuffix = opts.slot && opts.slot !== "1" ? `-${opts.slot === "h" ? "l" : `p${opts.slot}`}` : "";
const friendly = `${opts.slug ?? "portrait"}${slotSuffix}${ext}`;
return `/api/portrait/${safe(friendly)}?p=${encodeURIComponent(opts.path)}`;
}
export function categoryCoverUrl(filename: string): string {
return `/api/category-cover-file/${safe(filename)}`;
}
export function collectionCoverUrl(filename: string): string {
return `/api/collection-cover-file/${safe(filename)}`;
}
+195
View File
@@ -0,0 +1,195 @@
import "server-only";
import path from "node:path";
import { rawDb } from "@/lib/db/client";
import { clearAppSettingsCache } from "@/lib/db/appSettings";
const DB_PATH = path.join(process.cwd(), "data", "library.db");
const WIPE_ORDER = [
"actress_categories_map",
"collection_images",
"image_tags",
"image_genres",
"image_actresses",
"tags",
"tag_categories",
"genres",
"actresses",
"actress_categories",
"collections",
"series",
"labels",
"studios",
"images",
"app_settings",
];
const INSERT_ORDER = [
"studios",
"labels",
"series",
"actresses",
"genres",
"tag_categories",
"tags",
"actress_categories",
"images",
"collections",
"image_actresses",
"image_genres",
"image_tags",
"collection_images",
"actress_categories_map",
"app_settings",
];
function escIdent(s: string): string {
return `"${s.replace(/"/g, '""')}"`;
}
function tableColumns(schema: "main" | "restore", table: string): string[] {
try {
const rows = rawDb.prepare(`PRAGMA ${schema}.table_info(${escIdent(table)})`).all() as Array<{ name: string }>;
return rows.map((r) => r.name);
} catch {
return [];
}
}
export type ImportDbResult = {
ok: boolean;
counts: Record<string, number>;
errors: Array<{ table: string; message: string }>;
snapshotPath: string | null;
error?: string;
};
export async function importDatabaseTables(
tables: Record<string, unknown[]>,
): Promise<ImportDbResult> {
const counts: Record<string, number> = {};
const errors: Array<{ table: string; message: string }> = [];
let snapshotPath: string | null = null;
try {
const ts = new Date().toISOString().replace(/[:.]/g, "-");
snapshotPath = `${DB_PATH}.${ts}.bak`;
await rawDb.backup(snapshotPath);
} catch (e) {
return {
ok: false,
counts,
errors,
snapshotPath: null,
error: `Failed to snapshot DB before import: ${(e as Error).message}`,
};
}
rawDb.pragma("foreign_keys = OFF");
try {
const tx = rawDb.transaction(() => {
for (const t of WIPE_ORDER) {
try {
rawDb.prepare(`DELETE FROM ${escIdent(t)}`).run();
rawDb.prepare(`DELETE FROM sqlite_sequence WHERE name = ?`).run(t);
} catch (e) {
const msg = (e as Error).message ?? "";
if (!/no such table/i.test(msg)) {
throw new Error(`Wipe failed on ${t}: ${msg}`);
}
}
}
for (const t of INSERT_ORDER) {
const rows = tables[t];
if (!Array.isArray(rows) || rows.length === 0) {
counts[t] = 0;
continue;
}
const sample = rows[0] as Record<string, unknown>;
const cols = Object.keys(sample);
if (cols.length === 0) {
counts[t] = 0;
continue;
}
const colList = cols.map(escIdent).join(",");
const placeholders = cols.map(() => "?").join(",");
const stmt = rawDb.prepare(
`INSERT INTO ${escIdent(t)} (${colList}) VALUES (${placeholders})`,
);
let inserted = 0;
for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) {
const r = rows[rowIndex];
if (!r || typeof r !== "object") continue;
const row = r as Record<string, unknown>;
const values = cols.map((c) => {
const v = row[c];
if (v === undefined) return null;
if (typeof v === "boolean") return v ? 1 : 0;
return v as null | string | number | bigint | Buffer;
});
try {
stmt.run(...values);
inserted++;
} catch (e) {
const message = `Insert failed on ${t} row ${rowIndex + 1}: ${(e as Error).message}`;
errors.push({ table: t, message });
throw new Error(message);
}
}
counts[t] = inserted;
}
});
tx();
} catch (e) {
rawDb.pragma("foreign_keys = ON");
return {
ok: false,
counts,
errors,
snapshotPath,
error: (e as Error).message,
};
}
rawDb.pragma("foreign_keys = ON");
clearAppSettingsCache();
return { ok: true, counts, errors, snapshotPath };
}
export async function restoreDatabaseSnapshot(snapshotPath: string): Promise<void> {
rawDb.pragma("foreign_keys = OFF");
try {
rawDb.prepare("ATTACH DATABASE ? AS restore").run(snapshotPath);
const tx = rawDb.transaction(() => {
for (const t of WIPE_ORDER) {
try {
rawDb.prepare(`DELETE FROM ${escIdent(t)}`).run();
rawDb.prepare(`DELETE FROM sqlite_sequence WHERE name = ?`).run(t);
} catch (e) {
const msg = (e as Error).message ?? "";
if (!/no such table/i.test(msg)) throw e;
}
}
for (const t of INSERT_ORDER) {
const mainCols = tableColumns("main", t);
const restoreCols = new Set(tableColumns("restore", t));
const cols = mainCols.filter((c) => restoreCols.has(c));
if (cols.length === 0) continue;
const colList = cols.map(escIdent).join(",");
rawDb.prepare(`
INSERT INTO ${escIdent(t)} (${colList})
SELECT ${colList} FROM restore.${escIdent(t)}
`).run();
}
});
tx();
} finally {
try {
rawDb.prepare("DETACH DATABASE restore").run();
} catch {}
rawDb.pragma("foreign_keys = ON");
clearAppSettingsCache();
}
}
+13
View File
@@ -0,0 +1,13 @@
/**
* Single source of truth for product branding.
* Change values here and the rest of the app picks them up.
*/
export const BRAND = {
name: "Pinkudex",
tagline: "Your JAV index",
description: "A pink-filmera index for JAV cover art, cast, and the metadata behind every release.",
/** Prefix for any browser localStorage keys this app writes. */
storagePrefix: "pinkudex",
} as const;
export const storageKey = (key: string) => `${BRAND.storagePrefix}-${key}`;
+278
View File
@@ -0,0 +1,278 @@
import "server-only";
import { rawDb } from "./client";
import { isValidSort, type SortKey } from "../sort";
export interface AppSettings {
fadeTransitions: boolean;
fadeDurationMs: number;
/** When emptying the recycle bin (or hard-deleting), also remove the file from disk. */
purgeFilesOnDelete: boolean;
/** When true, delete sends to the recycle bin instead of removing immediately. */
useRecycleBin: boolean;
/** Auto-purge trashed items after this many days. 0 = never auto-purge. */
trashRetentionDays: number;
/** Auto-purge files in library/.superseded/ after this many days. 0 = never. */
supersededRetentionDays: number;
defaultSort: SortKey;
/** Hex color overriding --color-cyan / --color-cyan-glow. Empty string = use default. */
accentPrimary: string;
/** Hex color overriding --color-violet / --color-violet-glow. Empty string = use default. */
accentSecondary: string;
/** Number of cover columns in the masonry grid (24). */
gridColumns: number;
/** Number of cover columns in the portrait/front-only view (410). */
gridColumnsPortrait: number;
/** Cover grid page size — items fetched per page on `/` and per
* infinite-scroll append. */
coverPageSize: number;
/** Which layout the Settings drawer renders. "sidebar" = vertical nav
* with focused single-section content. "three-column" = everything
* visible in a 3-column wide layout. */
settingsLayout: "sidebar" | "three-column";
/** Absolute path to the main video library folder. Walked recursively,
* expected to follow the same letter-bucket structure as library/
* (e.g. D:\JAV\A-E\A\AOZ-200Z.mp4). Empty string disables. */
videoLibraryPath: string;
/** Additional video folders. These are also walked recursively, but
* the user typically points them at flat folders that hold files
* directly (e.g. E:\JAV\IBW-203.mp4). One absolute path per entry. */
videoExtraPaths: string[];
/** Folders scanned recursively for subtitle sidecars whose filename
* matches the playing video's stem or code. Empty by default — the
* player still finds same-folder sidecars without this. */
subtitleExtraPaths: string[];
/** Soft cap on data/subtitle-cache/ size in MB. When the cache grows
* beyond this, an LRU sweep deletes oldest-mtime entries until the
* total drops below 80% of the cap. 0 = unlimited. */
subtitleCacheLimitMb: number;
/** Playback path selector — controls how video files are streamed.
* - "off": always serve the original file directly via HTTP Range.
* - "always": always transcode through ffmpeg+NVENC HLS pipeline.
* - "auto-predicate": transcode only if a quick ffprobe says the file
* is H.264 with B-frames (the trigger profile for Chromium's sink
* reorder bug). Other codecs / no-B-frame H.264 → direct.
* - "auto-runtime": play direct first, measure dropped frames over
* a short pre-roll, decide per file, persist the decision. */
transcodeMode: "off" | "always" | "auto-predicate" | "auto-runtime";
/** Token-grammar suffix patterns used to classify video files into
* sequential parts. Patterns are matched at the end of the filename
* stem. `{N}` = digits (captured as part index), `{L}` = letter
* (A=1, B=2…), all other characters literal. Files in the same code
* group whose stem matches none of these patterns are treated as
* variants of the matched files (alternate encodes). */
partSuffixPatterns: string[];
/** WhisperJAV subtitle-generator integration. Empty cliPath disables
* the picker's Generate action. */
whisperjav: WhisperJavSettings;
/** How the pagination bar's Prev/Next/Jump buttons behave.
* - "url": always push a new URL, full page remount. Predictable.
* - "scroll": scroll within the loaded buffer; prefetch forward
* pages on demand; fall back to URL push only when the target is
* behind the SSR anchor. */
paginationMode: "url" | "scroll";
}
export type PaginationMode = AppSettings["paginationMode"];
export interface WhisperJavSettings {
/** Resolved CLI path. Empty string = feature disabled. */
cliPath: string;
quality: "fast" | "balanced" | "qwen";
sourceLanguage: "japanese" | "korean" | "chinese" | "english";
outputMode: "native" | "direct-to-english";
sensitivity: "conservative" | "balanced" | "aggressive";
outputLocation: "beside-video" | "data-folder";
/** When true, append --no-signature so generated subs don't include
* WhisperJAV's trailing technical signature cue. */
noSignature: boolean;
/** Days to keep failed / cancelled job temp directories. Successful
* job dirs are deleted immediately. 0 = keep forever. */
retentionDays: number;
}
export type TranscodeMode = AppSettings["transcodeMode"];
export const APP_SETTINGS_DEFAULTS: AppSettings = {
fadeTransitions: true,
fadeDurationMs: 400,
purgeFilesOnDelete: true,
useRecycleBin: true,
trashRetentionDays: 30,
supersededRetentionDays: 30,
defaultSort: "newest",
accentPrimary: "",
accentSecondary: "",
gridColumns: 3,
gridColumnsPortrait: 6,
coverPageSize: 100,
settingsLayout: "sidebar",
videoLibraryPath: "",
videoExtraPaths: [],
subtitleExtraPaths: [],
subtitleCacheLimitMb: 100,
transcodeMode: "off",
partSuffixPatterns: ["-cd{N}", ".part{N}", "_{N}", "_{L}"],
whisperjav: {
cliPath: "",
quality: "balanced",
sourceLanguage: "japanese",
outputMode: "native",
sensitivity: "balanced",
outputLocation: "beside-video",
noSignature: true,
retentionDays: 30,
},
paginationMode: "scroll",
};
const HEX_RE = /^#([0-9a-fA-F]{6})$/;
function decodeHex(raw: string): string | undefined {
if (raw === "") return "";
return HEX_RE.test(raw) ? raw.toLowerCase() : undefined;
}
// Pin cache to globalThis so multiple bundle copies of this module
// (Turbopack dev can produce more than one) share state — otherwise
// the server action that saves a setting writes to one cache while
// the queue worker keeps reading a stale value from another.
declare global {
// eslint-disable-next-line no-var
var __pinkudexAppSettingsCache: Map<string, string | undefined> | undefined;
}
const cache: Map<string, string | undefined> =
global.__pinkudexAppSettingsCache ?? (global.__pinkudexAppSettingsCache = new Map());
function getRaw(key: string): string | undefined {
if (cache.has(key)) return cache.get(key);
const row = rawDb.prepare(`SELECT value FROM app_settings WHERE key = ?`).get(key) as
| { value: string }
| undefined;
cache.set(key, row?.value);
return row?.value;
}
function setRaw(key: string, value: string): void {
rawDb.prepare(`
INSERT INTO app_settings (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
`).run(key, value);
cache.set(key, value);
}
const SERIALIZERS: { [K in keyof AppSettings]: { encode: (v: AppSettings[K]) => string; decode: (raw: string) => AppSettings[K] | undefined } } = {
fadeTransitions: { encode: (v) => (v ? "1" : "0"), decode: (r) => r === "1" ? true : r === "0" ? false : undefined },
fadeDurationMs: { encode: (v) => String(v), decode: (r) => { const n = Number(r); return Number.isFinite(n) ? n : undefined; } },
purgeFilesOnDelete: { encode: (v) => (v ? "1" : "0"), decode: (r) => r === "1" ? true : r === "0" ? false : undefined },
useRecycleBin: { encode: (v) => (v ? "1" : "0"), decode: (r) => r === "1" ? true : r === "0" ? false : undefined },
trashRetentionDays: { encode: (v) => String(v), decode: (r) => { const n = Number(r); return Number.isFinite(n) && n >= 0 ? n : undefined; } },
supersededRetentionDays: { encode: (v) => String(v), decode: (r) => { const n = Number(r); return Number.isFinite(n) && n >= 0 ? n : undefined; } },
defaultSort: { encode: (v) => v, decode: (r) => isValidSort(r) ? r : undefined },
accentPrimary: { encode: (v) => v, decode: decodeHex },
accentSecondary: { encode: (v) => v, decode: decodeHex },
gridColumns: { encode: (v) => String(v), decode: (r) => { const n = Number(r); return Number.isFinite(n) && n >= 2 && n <= 4 ? Math.round(n) : undefined; } },
gridColumnsPortrait: { encode: (v) => String(v), decode: (r) => { const n = Number(r); return Number.isFinite(n) && n >= 4 && n <= 10 ? Math.round(n) : undefined; } },
coverPageSize: { encode: (v) => String(v), decode: (r) => { const n = Number(r); return Number.isFinite(n) && n >= 25 && n <= 500 ? Math.round(n) : undefined; } },
settingsLayout: { encode: (v) => v, decode: (r) => r === "sidebar" || r === "three-column" ? r : undefined },
videoLibraryPath: { encode: (v) => v, decode: (r) => typeof r === "string" ? r : undefined },
videoExtraPaths: {
// Stored as newline-separated absolute paths so the value remains a
// plain string at the storage layer. Empty / blank lines are stripped.
encode: (v) => (v ?? []).filter((s) => s && s.trim()).join("\n"),
decode: (r) => typeof r === "string"
? r.split(/\r?\n/).map((s) => s.trim()).filter(Boolean)
: undefined,
},
subtitleExtraPaths: {
encode: (v) => (v ?? []).filter((s) => s && s.trim()).join("\n"),
decode: (r) => typeof r === "string"
? r.split(/\r?\n/).map((s) => s.trim()).filter(Boolean)
: undefined,
},
subtitleCacheLimitMb: {
encode: (v) => String(v),
decode: (r) => {
const n = Number(r);
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : undefined;
},
},
transcodeMode: {
encode: (v) => v,
decode: (r) => {
if (r === "off" || r === "always" || r === "auto-predicate" || r === "auto-runtime") return r;
// Migrate legacy boolean values from the old `transcodePlayback` setting.
if (r === "1") return "always";
if (r === "0") return "off";
return undefined;
},
},
partSuffixPatterns: {
// Newline-separated to keep the storage value a plain string. Empty
// / blank lines are stripped on decode.
encode: (v) => (v ?? []).map((s) => s ?? "").map((s) => s.trim()).filter(Boolean).join("\n"),
decode: (r) => typeof r === "string"
? r.split(/\r?\n/).map((s) => s.trim()).filter(Boolean)
: undefined,
},
paginationMode: {
encode: (v) => v,
decode: (r) => {
if (r === "url" || r === "scroll") return r;
// Migrate any persisted "split" rows to scroll (the default).
if (r === "split") return "scroll";
return undefined;
},
},
whisperjav: {
encode: (v) => JSON.stringify(v ?? APP_SETTINGS_DEFAULTS.whisperjav),
decode: (r) => {
if (typeof r !== "string") return undefined;
try {
const parsed = JSON.parse(r) as Partial<WhisperJavSettings>;
// Merge with defaults so older rows missing a key still resolve.
return { ...APP_SETTINGS_DEFAULTS.whisperjav, ...parsed };
} catch {
return undefined;
}
},
},
};
export function getAppSetting<K extends keyof AppSettings>(key: K): AppSettings[K] {
const raw = getRaw(key);
const decoded = raw == null ? undefined : SERIALIZERS[key].decode(raw);
return (decoded ?? APP_SETTINGS_DEFAULTS[key]) as AppSettings[K];
}
export function setAppSetting<K extends keyof AppSettings>(key: K, value: AppSettings[K]): void {
setRaw(key, SERIALIZERS[key].encode(value));
}
export function getAllAppSettings(): AppSettings {
return {
fadeTransitions: getAppSetting("fadeTransitions"),
fadeDurationMs: getAppSetting("fadeDurationMs"),
purgeFilesOnDelete: getAppSetting("purgeFilesOnDelete"),
useRecycleBin: getAppSetting("useRecycleBin"),
trashRetentionDays: getAppSetting("trashRetentionDays"),
supersededRetentionDays: getAppSetting("supersededRetentionDays"),
defaultSort: getAppSetting("defaultSort"),
accentPrimary: getAppSetting("accentPrimary"),
accentSecondary: getAppSetting("accentSecondary"),
gridColumns: getAppSetting("gridColumns"),
gridColumnsPortrait: getAppSetting("gridColumnsPortrait"),
coverPageSize: getAppSetting("coverPageSize"),
settingsLayout: getAppSetting("settingsLayout"),
videoLibraryPath: getAppSetting("videoLibraryPath"),
videoExtraPaths: getAppSetting("videoExtraPaths"),
subtitleExtraPaths: getAppSetting("subtitleExtraPaths"),
subtitleCacheLimitMb: getAppSetting("subtitleCacheLimitMb"),
transcodeMode: getAppSetting("transcodeMode"),
partSuffixPatterns: getAppSetting("partSuffixPatterns"),
whisperjav: getAppSetting("whisperjav"),
paginationMode: getAppSetting("paginationMode"),
};
}
export function clearAppSettingsCache(): void {
cache.clear();
}
+539
View File
@@ -0,0 +1,539 @@
import "server-only";
import Database from "better-sqlite3";
import { drizzle } from "drizzle-orm/better-sqlite3";
import * as schema from "./schema";
import path from "node:path";
import fs from "node:fs";
import { safeJoin } from "../safePath";
const DB_PATH = path.join(process.cwd(), "data", "library.db");
fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
fs.mkdirSync(path.join(process.cwd(), "data", "thumbs"), { recursive: true });
fs.mkdirSync(path.join(process.cwd(), "data", "portraits"), { recursive: true });
fs.mkdirSync(path.join(process.cwd(), "library"), { recursive: true });
declare global {
// eslint-disable-next-line no-var
var __sqlite: Database.Database | undefined;
}
const sqlite = global.__sqlite ?? new Database(DB_PATH);
if (!global.__sqlite) {
sqlite.pragma("journal_mode = WAL");
sqlite.pragma("foreign_keys = ON");
sqlite.pragma("synchronous = NORMAL");
global.__sqlite = sqlite;
}
bootstrap(sqlite);
function bootstrap(db: Database.Database) {
db.exec(`
CREATE TABLE IF NOT EXISTS images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
filename TEXT NOT NULL,
rel_path TEXT NOT NULL UNIQUE,
thumb_path TEXT NOT NULL,
sha256 TEXT NOT NULL UNIQUE,
width INTEGER NOT NULL,
height INTEGER NOT NULL,
bytes INTEGER NOT NULL,
raw_metadata TEXT,
created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
imported_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
deleted_at INTEGER,
parent_image_id INTEGER REFERENCES images(id) ON DELETE CASCADE,
code TEXT,
title TEXT,
release_date TEXT,
runtime_min INTEGER,
director TEXT,
studio_id INTEGER REFERENCES studios(id) ON DELETE SET NULL,
label_id INTEGER REFERENCES labels(id) ON DELETE SET NULL,
series_id INTEGER REFERENCES series(id) ON DELETE SET NULL,
rating INTEGER,
watched INTEGER NOT NULL DEFAULT 0,
is_vip INTEGER NOT NULL DEFAULT 0,
is_favorite INTEGER NOT NULL DEFAULT 0,
is_owned INTEGER NOT NULL DEFAULT 0,
notes TEXT,
phash TEXT
);
CREATE INDEX IF NOT EXISTS images_created_idx ON images(created_at);
CREATE INDEX IF NOT EXISTS images_deleted_idx ON images(deleted_at);
CREATE INDEX IF NOT EXISTS images_parent_idx ON images(parent_image_id);
CREATE INDEX IF NOT EXISTS images_code_idx ON images(code);
CREATE INDEX IF NOT EXISTS images_studio_idx ON images(studio_id);
CREATE INDEX IF NOT EXISTS images_label_idx ON images(label_id);
CREATE INDEX IF NOT EXISTS images_series_idx ON images(series_id);
CREATE TABLE IF NOT EXISTS studios (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE,
notes TEXT
);
CREATE TABLE IF NOT EXISTS labels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE,
notes TEXT
);
CREATE TABLE IF NOT EXISTS series (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE,
notes TEXT
);
CREATE TABLE IF NOT EXISTS actresses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE,
alt_names TEXT,
notes TEXT,
portrait_path TEXT,
portrait_zoom REAL NOT NULL DEFAULT 1,
portrait_offset_x REAL NOT NULL DEFAULT 0,
portrait_offset_y REAL NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS genres (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE
);
CREATE TABLE IF NOT EXISTS image_actresses (
image_id INTEGER NOT NULL REFERENCES images(id) ON DELETE CASCADE,
actress_id INTEGER NOT NULL REFERENCES actresses(id) ON DELETE CASCADE,
PRIMARY KEY (image_id, actress_id)
);
CREATE INDEX IF NOT EXISTS image_actresses_actress_idx ON image_actresses(actress_id);
CREATE TABLE IF NOT EXISTS image_genres (
image_id INTEGER NOT NULL REFERENCES images(id) ON DELETE CASCADE,
genre_id INTEGER NOT NULL REFERENCES genres(id) ON DELETE CASCADE,
PRIMARY KEY (image_id, genre_id)
);
CREATE INDEX IF NOT EXISTS image_genres_genre_idx ON image_genres(genre_id);
CREATE TABLE IF NOT EXISTS tag_categories (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE,
color TEXT,
description TEXT,
cover_portrait_path TEXT,
cover_portrait_zoom REAL NOT NULL DEFAULT 1,
cover_portrait_offset_x REAL NOT NULL DEFAULT 0,
cover_portrait_offset_y REAL NOT NULL DEFAULT 0,
cover_landscape_path TEXT,
cover_landscape_zoom REAL NOT NULL DEFAULT 1,
cover_landscape_offset_x REAL NOT NULL DEFAULT 0,
cover_landscape_offset_y REAL NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
color TEXT,
category_id INTEGER REFERENCES tag_categories(id) ON DELETE SET NULL,
last_used_at INTEGER NOT NULL DEFAULT 0
);
-- tags_category_idx is created AFTER the idempotent ALTER below, so
-- it doesn't fire on an old DB whose tags table predates category_id.
CREATE TABLE IF NOT EXISTS image_tags (
image_id INTEGER NOT NULL REFERENCES images(id) ON DELETE CASCADE,
tag_id INTEGER NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
PRIMARY KEY (image_id, tag_id)
);
CREATE INDEX IF NOT EXISTS image_tags_tag_idx ON image_tags(tag_id);
CREATE TABLE IF NOT EXISTS collections (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
slug TEXT NOT NULL UNIQUE,
description TEXT,
cover_image_id INTEGER REFERENCES images(id) ON DELETE SET NULL,
created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
position INTEGER NOT NULL DEFAULT 0,
last_used_at INTEGER NOT NULL DEFAULT 0,
cover_portrait_path TEXT,
cover_portrait_zoom REAL NOT NULL DEFAULT 1,
cover_portrait_offset_x REAL NOT NULL DEFAULT 0,
cover_portrait_offset_y REAL NOT NULL DEFAULT 0,
cover_landscape_path TEXT,
cover_landscape_zoom REAL NOT NULL DEFAULT 1,
cover_landscape_offset_x REAL NOT NULL DEFAULT 0,
cover_landscape_offset_y REAL NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS collection_images (
collection_id INTEGER NOT NULL REFERENCES collections(id) ON DELETE CASCADE,
image_id INTEGER NOT NULL REFERENCES images(id) ON DELETE CASCADE,
position INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (collection_id, image_id)
);
CREATE TABLE IF NOT EXISTS actress_categories (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL UNIQUE,
color TEXT,
icon TEXT,
priority INTEGER NOT NULL DEFAULT 0,
builtin INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS actress_categories_map (
actress_id INTEGER NOT NULL REFERENCES actresses(id) ON DELETE CASCADE,
category_id INTEGER NOT NULL REFERENCES actress_categories(id) ON DELETE CASCADE,
PRIMARY KEY (actress_id, category_id)
);
CREATE INDEX IF NOT EXISTS actress_cat_map_cat_idx ON actress_categories_map(category_id);
CREATE TABLE IF NOT EXISTS app_settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS video_metadata (
abs_path TEXT PRIMARY KEY,
rel_path TEXT NOT NULL,
code TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
mtime_ms REAL NOT NULL,
probed_at INTEGER,
probe_error TEXT,
duration_sec REAL,
video_codec TEXT,
video_b_frames INTEGER,
width INTEGER,
height INTEGER,
video_bitrate INTEGER,
playback_mode TEXT
);
CREATE INDEX IF NOT EXISTS video_metadata_code_idx ON video_metadata(code);
CREATE TABLE IF NOT EXISTS whisperjav_jobs (
id TEXT PRIMARY KEY,
code TEXT NOT NULL,
video_abs TEXT NOT NULL,
job_dir TEXT NOT NULL,
target_subtitle_path TEXT,
status TEXT NOT NULL CHECK(status IN ('queued','running','completed','warning','failed','cancelled')),
enqueued_at INTEGER NOT NULL,
started_at INTEGER,
ended_at INTEGER,
exit_code INTEGER,
error TEXT,
stage TEXT,
stage_index INTEGER,
stage_total INTEGER,
cue_count INTEGER,
cli_args TEXT NOT NULL,
log_path TEXT NOT NULL,
stats_path TEXT,
video_duration_sec REAL,
mode TEXT
);
CREATE INDEX IF NOT EXISTS whisperjav_jobs_code_idx ON whisperjav_jobs(code);
CREATE INDEX IF NOT EXISTS whisperjav_jobs_status_idx ON whisperjav_jobs(status);
CREATE INDEX IF NOT EXISTS whisperjav_jobs_enqueued_idx ON whisperjav_jobs(enqueued_at);
-- User-attached subtitle files that live outside the indexed roots
-- and subtitleExtraPaths. Browse... in the player records the pick
-- here so the entry survives modal close / server restart.
CREATE TABLE IF NOT EXISTS manual_subtitles (
code TEXT NOT NULL,
part_idx INTEGER NOT NULL,
abs_path TEXT NOT NULL,
attached_at INTEGER NOT NULL,
PRIMARY KEY (code, part_idx, abs_path)
);
CREATE INDEX IF NOT EXISTS manual_subtitles_code_idx ON manual_subtitles(code);
CREATE INDEX IF NOT EXISTS manual_subtitles_abs_idx ON manual_subtitles(abs_path);
CREATE VIRTUAL TABLE IF NOT EXISTS covers_fts USING fts5(
code, title, director, notes,
content='images', content_rowid='id',
tokenize='porter unicode61'
);
CREATE TRIGGER IF NOT EXISTS covers_ai AFTER INSERT ON images BEGIN
INSERT INTO covers_fts(rowid, code, title, director, notes)
VALUES (new.id, COALESCE(new.code, ''), COALESCE(new.title, ''), COALESCE(new.director, ''), COALESCE(new.notes, ''));
END;
CREATE TRIGGER IF NOT EXISTS covers_ad AFTER DELETE ON images BEGIN
INSERT INTO covers_fts(covers_fts, rowid, code, title, director, notes)
VALUES ('delete', old.id, COALESCE(old.code, ''), COALESCE(old.title, ''), COALESCE(old.director, ''), COALESCE(old.notes, ''));
END;
CREATE TRIGGER IF NOT EXISTS covers_au AFTER UPDATE ON images BEGIN
INSERT INTO covers_fts(covers_fts, rowid, code, title, director, notes)
VALUES ('delete', old.id, COALESCE(old.code, ''), COALESCE(old.title, ''), COALESCE(old.director, ''), COALESCE(old.notes, ''));
INSERT INTO covers_fts(rowid, code, title, director, notes)
VALUES (new.id, COALESCE(new.code, ''), COALESCE(new.title, ''), COALESCE(new.director, ''), COALESCE(new.notes, ''));
END;
`);
// Existing databases may have images that predate the FTS table/triggers.
// Rebuild once per process bootstrap so metadata search is complete.
try {
db.prepare(`INSERT INTO covers_fts(covers_fts) VALUES ('rebuild')`).run();
} catch {}
// Seed built-in actress categories (idempotent — only inserts if absent).
const seedCat = db.prepare(`
INSERT OR IGNORE INTO actress_categories (name, slug, color, icon, priority, builtin)
VALUES (?, ?, ?, ?, ?, 1)
`);
seedCat.run("Favorite", "favorite", "#fbbf24", "star", 100);
seedCat.run("VIP", "vip", "#22d3ee", "gem", 90);
// Idempotent ALTERs for columns added after initial release.
const actressCols = db.prepare(`PRAGMA table_info(actresses)`).all() as Array<{ name: string }>;
const hasCol = (n: string) => actressCols.some((c) => c.name === n);
if (!hasCol("portrait_path")) db.exec(`ALTER TABLE actresses ADD COLUMN portrait_path TEXT`);
if (!hasCol("portrait_zoom")) db.exec(`ALTER TABLE actresses ADD COLUMN portrait_zoom REAL NOT NULL DEFAULT 1`);
if (!hasCol("portrait_offset_x")) db.exec(`ALTER TABLE actresses ADD COLUMN portrait_offset_x REAL NOT NULL DEFAULT 0`);
if (!hasCol("portrait_offset_y")) db.exec(`ALTER TABLE actresses ADD COLUMN portrait_offset_y REAL NOT NULL DEFAULT 0`);
for (const slot of ["2", "3", "4", "h"]) {
if (!hasCol(`portrait${slot}_path`)) db.exec(`ALTER TABLE actresses ADD COLUMN portrait${slot}_path TEXT`);
if (!hasCol(`portrait${slot}_zoom`)) db.exec(`ALTER TABLE actresses ADD COLUMN portrait${slot}_zoom REAL NOT NULL DEFAULT 1`);
if (!hasCol(`portrait${slot}_offset_x`)) db.exec(`ALTER TABLE actresses ADD COLUMN portrait${slot}_offset_x REAL NOT NULL DEFAULT 0`);
if (!hasCol(`portrait${slot}_offset_y`)) db.exec(`ALTER TABLE actresses ADD COLUMN portrait${slot}_offset_y REAL NOT NULL DEFAULT 0`);
}
if (!hasCol("born_on")) db.exec(`ALTER TABLE actresses ADD COLUMN born_on TEXT`);
if (!hasCol("height_cm")) db.exec(`ALTER TABLE actresses ADD COLUMN height_cm INTEGER`);
if (!hasCol("weight_kg")) db.exec(`ALTER TABLE actresses ADD COLUMN weight_kg INTEGER`);
if (!hasCol("cup_size")) db.exec(`ALTER TABLE actresses ADD COLUMN cup_size TEXT`);
// VIP / Favorite toggles on covers (separate from the actress categories of the same name).
const imageCols = db.prepare(`PRAGMA table_info(images)`).all() as Array<{ name: string }>;
const hasImgCol = (n: string) => imageCols.some((c) => c.name === n);
if (!hasImgCol("is_vip")) db.exec(`ALTER TABLE images ADD COLUMN is_vip INTEGER NOT NULL DEFAULT 0`);
if (!hasImgCol("is_favorite")) db.exec(`ALTER TABLE images ADD COLUMN is_favorite INTEGER NOT NULL DEFAULT 0`);
if (!hasImgCol("is_owned")) db.exec(`ALTER TABLE images ADD COLUMN is_owned INTEGER NOT NULL DEFAULT 0`);
// Set by the video scanner whenever the on-disk index is rebuilt — see lib/video/index.ts.
if (!hasImgCol("has_video")) {
db.exec(`ALTER TABLE images ADD COLUMN has_video INTEGER NOT NULL DEFAULT 0`);
db.exec(`CREATE INDEX IF NOT EXISTS images_has_video_idx ON images(has_video)`);
}
// Mirrors the on-disk subtitle index — true when at least one sidecar
// (.srt/.vtt/.ass/.ssa) exists for the code in any of: the video's
// folder, configured subtitleExtraPaths, or data/generated-subtitles/.
// Embedded streams are NOT counted (cheap-only signal).
if (!hasImgCol("has_subtitle")) {
db.exec(`ALTER TABLE images ADD COLUMN has_subtitle INTEGER NOT NULL DEFAULT 0`);
db.exec(`CREATE INDEX IF NOT EXISTS images_has_subtitle_idx ON images(has_subtitle)`);
}
// Lazy-probed video metadata used by playback auto-modes. Populated on
// first play of a file; never written by the scanner.
if (!hasImgCol("video_codec")) db.exec(`ALTER TABLE images ADD COLUMN video_codec TEXT`);
if (!hasImgCol("video_b_frames")) db.exec(`ALTER TABLE images ADD COLUMN video_b_frames INTEGER`);
if (!hasImgCol("playback_mode")) db.exec(`ALTER TABLE images ADD COLUMN playback_mode TEXT`);
const videoMetaCols = db.prepare(`PRAGMA table_info(video_metadata)`).all() as Array<{ name: string }>;
const hasVideoMetaCol = (n: string) => videoMetaCols.some((c) => c.name === n);
if (!hasVideoMetaCol("rel_path")) db.exec(`ALTER TABLE video_metadata ADD COLUMN rel_path TEXT NOT NULL DEFAULT ''`);
if (!hasVideoMetaCol("code")) db.exec(`ALTER TABLE video_metadata ADD COLUMN code TEXT NOT NULL DEFAULT ''`);
if (!hasVideoMetaCol("size_bytes")) db.exec(`ALTER TABLE video_metadata ADD COLUMN size_bytes INTEGER NOT NULL DEFAULT 0`);
if (!hasVideoMetaCol("mtime_ms")) db.exec(`ALTER TABLE video_metadata ADD COLUMN mtime_ms REAL NOT NULL DEFAULT 0`);
if (!hasVideoMetaCol("probed_at")) db.exec(`ALTER TABLE video_metadata ADD COLUMN probed_at INTEGER`);
if (!hasVideoMetaCol("probe_error")) db.exec(`ALTER TABLE video_metadata ADD COLUMN probe_error TEXT`);
if (!hasVideoMetaCol("duration_sec")) db.exec(`ALTER TABLE video_metadata ADD COLUMN duration_sec REAL`);
if (!hasVideoMetaCol("video_codec")) db.exec(`ALTER TABLE video_metadata ADD COLUMN video_codec TEXT`);
if (!hasVideoMetaCol("video_b_frames")) db.exec(`ALTER TABLE video_metadata ADD COLUMN video_b_frames INTEGER`);
if (!hasVideoMetaCol("width")) db.exec(`ALTER TABLE video_metadata ADD COLUMN width INTEGER`);
if (!hasVideoMetaCol("height")) db.exec(`ALTER TABLE video_metadata ADD COLUMN height INTEGER`);
if (!hasVideoMetaCol("video_bitrate")) db.exec(`ALTER TABLE video_metadata ADD COLUMN video_bitrate INTEGER`);
if (!hasVideoMetaCol("playback_mode")) db.exec(`ALTER TABLE video_metadata ADD COLUMN playback_mode TEXT`);
if (!hasVideoMetaCol("part_kind")) db.exec(`ALTER TABLE video_metadata ADD COLUMN part_kind TEXT`);
if (!hasVideoMetaCol("part_index")) db.exec(`ALTER TABLE video_metadata ADD COLUMN part_index INTEGER`);
if (!hasVideoMetaCol("variant_group")) db.exec(`ALTER TABLE video_metadata ADD COLUMN variant_group TEXT`);
// dir_path enables incremental rescan: reuse cached rows for any
// directory whose mtime hasn't changed since last scan. Backfilled
// below for any pre-existing rows.
if (!hasVideoMetaCol("dir_path")) db.exec(`ALTER TABLE video_metadata ADD COLUMN dir_path TEXT NOT NULL DEFAULT ''`);
db.exec(`CREATE INDEX IF NOT EXISTS video_metadata_code_idx ON video_metadata(code)`);
db.exec(`CREATE INDEX IF NOT EXISTS video_metadata_dir_idx ON video_metadata(dir_path)`);
// Per-directory mtime cache. On rescan, dirs whose stat mtime
// matches the stored value are treated as unchanged and their cached
// file rows are reused without readdir/stat per file.
db.exec(`
CREATE TABLE IF NOT EXISTS video_dir_mtimes (
abs_dir TEXT PRIMARY KEY,
mtime_ms REAL NOT NULL,
last_seen_at INTEGER NOT NULL
);
`);
// Backfill dir_path for existing rows that predate this column.
// Cheap: one UPDATE per missing row, derived in SQL via rtrim/instr.
// Do nothing if there's nothing to fill (default '' marker).
const missing = db.prepare(`SELECT COUNT(*) AS n FROM video_metadata WHERE dir_path = ''`).get() as { n: number };
if (missing.n > 0) {
type Row = { abs_path: string };
const rows = db.prepare(`SELECT abs_path FROM video_metadata WHERE dir_path = ''`).all() as Row[];
const upd = db.prepare(`UPDATE video_metadata SET dir_path = ? WHERE abs_path = ?`);
const tx = db.transaction(() => {
for (const r of rows) {
const last = Math.max(r.abs_path.lastIndexOf("/"), r.abs_path.lastIndexOf("\\"));
const dir = last >= 0 ? r.abs_path.slice(0, last) : "";
upd.run(dir, r.abs_path);
}
});
tx();
}
// whisperjav_jobs: ETA fields added in v1.1 — idempotent migration.
const wjCols = db.prepare(`PRAGMA table_info(whisperjav_jobs)`).all() as Array<{ name: string }>;
const hasWjCol = (n: string) => wjCols.some((c) => c.name === n);
if (!hasWjCol("video_duration_sec")) db.exec(`ALTER TABLE whisperjav_jobs ADD COLUMN video_duration_sec REAL`);
if (!hasWjCol("mode")) db.exec(`ALTER TABLE whisperjav_jobs ADD COLUMN mode TEXT`);
db.exec(`CREATE INDEX IF NOT EXISTS whisperjav_jobs_mode_idx ON whisperjav_jobs(mode)`);
// Perceptual dHash (16-char hex, 64 bits) for near-duplicate detection.
// Backfilled lazily by the maintenance scanner.
if (!hasImgCol("phash")) db.exec(`ALTER TABLE images ADD COLUMN phash TEXT`);
// Recency tracking on tags + collections so the context menu can
// surface "recent" chips. Updated whenever the entity is attached to
// an image. Default backfill = current time so existing rows show up.
const tagCols2 = db.prepare(`PRAGMA table_info(tags)`).all() as Array<{ name: string }>;
if (!tagCols2.some((c) => c.name === "last_used_at")) {
db.exec(`ALTER TABLE tags ADD COLUMN last_used_at INTEGER NOT NULL DEFAULT 0`);
db.exec(`UPDATE tags SET last_used_at = (unixepoch() * 1000)`);
}
const collectionCols = db.prepare(`PRAGMA table_info(collections)`).all() as Array<{ name: string }>;
if (!collectionCols.some((c) => c.name === "last_used_at")) {
db.exec(`ALTER TABLE collections ADD COLUMN last_used_at INTEGER NOT NULL DEFAULT 0`);
db.exec(`UPDATE collections SET last_used_at = COALESCE(created_at, unixepoch() * 1000)`);
}
// Manual reorder support for the collections index page. Backfill
// positions from created_at so existing libraries get a sensible
// initial order on first launch with the new schema.
const colCols = db.prepare(`PRAGMA table_info(collections)`).all() as Array<{ name: string }>;
if (!colCols.some((c) => c.name === "position")) {
db.exec(`ALTER TABLE collections ADD COLUMN position INTEGER NOT NULL DEFAULT 0`);
const rows = db.prepare(`SELECT id FROM collections ORDER BY created_at ASC, id ASC`).all() as Array<{ id: number }>;
const update = db.prepare(`UPDATE collections SET position = ? WHERE id = ?`);
for (let i = 0; i < rows.length; i++) update.run(i, rows[i].id);
}
// Tag categories: umbrellas grouping related tags (e.g. "BDSM" containing
// bondage / shibari / cuffs). Each tag belongs to at most one category.
const tagCols = db.prepare(`PRAGMA table_info(tags)`).all() as Array<{ name: string }>;
if (!tagCols.some((c) => c.name === "category_id")) {
db.exec(`ALTER TABLE tags ADD COLUMN category_id INTEGER REFERENCES tag_categories(id) ON DELETE SET NULL`);
}
// Create the index unconditionally — both the fresh-DB CREATE TABLE path
// and the migrated path need it, and IF NOT EXISTS makes it idempotent.
db.exec(`CREATE INDEX IF NOT EXISTS tags_category_idx ON tags(category_id)`);
// Tag-category cover art: separate portrait + landscape slots with
// pan/zoom transforms, mirroring the actress-portrait shape. Files
// live in data/category-covers/.
const catCols = db.prepare(`PRAGMA table_info(tag_categories)`).all() as Array<{ name: string }>;
const hasCatCol = (n: string) => catCols.some((c) => c.name === n);
if (!hasCatCol("cover_portrait_path")) db.exec(`ALTER TABLE tag_categories ADD COLUMN cover_portrait_path TEXT`);
if (!hasCatCol("cover_portrait_zoom")) db.exec(`ALTER TABLE tag_categories ADD COLUMN cover_portrait_zoom REAL NOT NULL DEFAULT 1`);
if (!hasCatCol("cover_portrait_offset_x")) db.exec(`ALTER TABLE tag_categories ADD COLUMN cover_portrait_offset_x REAL NOT NULL DEFAULT 0`);
if (!hasCatCol("cover_portrait_offset_y")) db.exec(`ALTER TABLE tag_categories ADD COLUMN cover_portrait_offset_y REAL NOT NULL DEFAULT 0`);
if (!hasCatCol("cover_landscape_path")) db.exec(`ALTER TABLE tag_categories ADD COLUMN cover_landscape_path TEXT`);
if (!hasCatCol("cover_landscape_zoom")) db.exec(`ALTER TABLE tag_categories ADD COLUMN cover_landscape_zoom REAL NOT NULL DEFAULT 1`);
if (!hasCatCol("cover_landscape_offset_x")) db.exec(`ALTER TABLE tag_categories ADD COLUMN cover_landscape_offset_x REAL NOT NULL DEFAULT 0`);
if (!hasCatCol("cover_landscape_offset_y")) db.exec(`ALTER TABLE tag_categories ADD COLUMN cover_landscape_offset_y REAL NOT NULL DEFAULT 0`);
fs.mkdirSync(path.join(process.cwd(), "data", "category-covers"), { recursive: true });
// Collection cover art: same shape as tag_categories — separate
// portrait + landscape slots with pan/zoom transforms. Files live in
// data/collection-covers/.
const collCols = db.prepare(`PRAGMA table_info(collections)`).all() as Array<{ name: string }>;
const hasCollCol = (n: string) => collCols.some((c) => c.name === n);
if (!hasCollCol("cover_portrait_path")) db.exec(`ALTER TABLE collections ADD COLUMN cover_portrait_path TEXT`);
if (!hasCollCol("cover_portrait_zoom")) db.exec(`ALTER TABLE collections ADD COLUMN cover_portrait_zoom REAL NOT NULL DEFAULT 1`);
if (!hasCollCol("cover_portrait_offset_x")) db.exec(`ALTER TABLE collections ADD COLUMN cover_portrait_offset_x REAL NOT NULL DEFAULT 0`);
if (!hasCollCol("cover_portrait_offset_y")) db.exec(`ALTER TABLE collections ADD COLUMN cover_portrait_offset_y REAL NOT NULL DEFAULT 0`);
if (!hasCollCol("cover_landscape_path")) db.exec(`ALTER TABLE collections ADD COLUMN cover_landscape_path TEXT`);
if (!hasCollCol("cover_landscape_zoom")) db.exec(`ALTER TABLE collections ADD COLUMN cover_landscape_zoom REAL NOT NULL DEFAULT 1`);
if (!hasCollCol("cover_landscape_offset_x")) db.exec(`ALTER TABLE collections ADD COLUMN cover_landscape_offset_x REAL NOT NULL DEFAULT 0`);
if (!hasCollCol("cover_landscape_offset_y")) db.exec(`ALTER TABLE collections ADD COLUMN cover_landscape_offset_y REAL NOT NULL DEFAULT 0`);
fs.mkdirSync(path.join(process.cwd(), "data", "collection-covers"), { recursive: true });
// Auto-purge expired trash. Reads retention from app_settings; bails if 0
// (forever) or no rows are old enough yet.
const retentionRow = db.prepare(`SELECT value FROM app_settings WHERE key = 'trashRetentionDays'`).get() as { value: string } | undefined;
const retentionDays = retentionRow ? Number(retentionRow.value) : 30;
if (Number.isFinite(retentionDays) && retentionDays > 0) {
const cutoff = Date.now() - retentionDays * 86400_000;
const stale = db.prepare(`
WITH targets AS (
SELECT id FROM images WHERE deleted_at IS NOT NULL AND deleted_at < ?
)
SELECT id, rel_path, thumb_path FROM images
WHERE id IN (SELECT id FROM targets)
OR parent_image_id IN (SELECT id FROM targets)
`).all(cutoff) as Array<{ id: number; rel_path: string; thumb_path: string }>;
if (stale.length > 0) {
const purgeRow = db.prepare(`SELECT value FROM app_settings WHERE key = 'purgeFilesOnDelete'`).get() as { value: string } | undefined;
const purgeFiles = purgeRow ? purgeRow.value === "1" : true;
if (purgeFiles) {
const libRoot = path.join(process.cwd(), "library");
const thumbRoot = path.join(process.cwd(), "data", "thumbs");
for (const r of stale) {
const fileAbs = safeJoin(libRoot, r.rel_path);
const thumbAbs = safeJoin(thumbRoot, r.thumb_path);
try { if (fileAbs) fs.rmSync(fileAbs, { force: true }); } catch {}
try { if (thumbAbs) fs.rmSync(thumbAbs, { force: true }); } catch {}
}
}
const placeholders = stale.map(() => "?").join(",");
db.prepare(`DELETE FROM images WHERE id IN (${placeholders})`).run(...stale.map((r) => r.id));
}
}
// Auto-purge old .superseded/ files. These are recovery snapshots
// written by the collision-replace path; once N days have passed
// without rolling back, they're safe to drop. 0 = keep forever.
const supersededRow = db.prepare(`SELECT value FROM app_settings WHERE key = 'supersededRetentionDays'`).get() as { value: string } | undefined;
const supersededDays = supersededRow ? Number(supersededRow.value) : 30;
if (Number.isFinite(supersededDays) && supersededDays > 0) {
const supersededRoot = path.join(process.cwd(), "library", ".superseded");
if (fs.existsSync(supersededRoot)) {
const cutoff = Date.now() - supersededDays * 86400_000;
try {
const entries = fs.readdirSync(supersededRoot, { withFileTypes: true });
for (const e of entries) {
if (!e.isFile()) continue;
const abs = path.join(supersededRoot, e.name);
try {
const stat = fs.statSync(abs);
if (stat.mtimeMs < cutoff) {
fs.rmSync(abs, { force: true });
}
} catch {}
}
} catch {}
}
}
}
function slugify(s: string): string {
return s.toLowerCase().normalize("NFKD").replace(/[̀-ͯ]/g, "")
.replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "untitled";
}
/** Pick a unique slug for a row in a slug-bearing table (collections, studios, labels, series, actresses, genres). */
export function uniqueSlug(database: Database.Database, table: string, name: string, excludeId?: number): string {
const base = slugify(name);
let slug = base;
let i = 1;
const stmt = database.prepare(`SELECT 1 FROM ${table} WHERE slug = ? AND id != ? LIMIT 1`);
while (stmt.get(slug, excludeId ?? -1)) {
i++;
slug = `${base}-${i}`;
}
return slug;
}
export const db = drizzle(sqlite, { schema });
export const rawDb = sqlite;
+1290
View File
File diff suppressed because it is too large Load Diff
+252
View File
@@ -0,0 +1,252 @@
import { sqliteTable, integer, text, real, primaryKey, index } from "drizzle-orm/sqlite-core";
import { relations, sql } from "drizzle-orm";
/**
* The asset table — each row is one image on disk. For "front cover" rows the
* cover-metadata columns are populated; for back covers / extra stills, the
* row points at its parent via parent_image_id and its own metadata is null.
*/
export const images = sqliteTable("images", {
id: integer("id").primaryKey({ autoIncrement: true }),
filename: text("filename").notNull(),
relPath: text("rel_path").notNull().unique(),
thumbPath: text("thumb_path").notNull(),
sha256: text("sha256").notNull().unique(),
width: integer("width").notNull(),
height: integer("height").notNull(),
bytes: integer("bytes").notNull(),
phash: text("phash"),
rawMetadata: text("raw_metadata"),
createdAt: integer("created_at").notNull().default(sql`(unixepoch() * 1000)`),
importedAt: integer("imported_at").notNull().default(sql`(unixepoch() * 1000)`),
deletedAt: integer("deleted_at"),
// Non-null = this is an attached image (e.g. back cover / still) of another row.
parentImageId: integer("parent_image_id").references((): any => images.id, { onDelete: "cascade" }),
// Cover metadata — only meaningful when parentImageId IS NULL.
code: text("code"),
title: text("title"),
releaseDate: text("release_date"), // ISO yyyy-mm-dd
runtimeMin: integer("runtime_min"),
director: text("director"),
studioId: integer("studio_id").references((): any => studios.id, { onDelete: "set null" }),
labelId: integer("label_id").references((): any => labels.id, { onDelete: "set null" }),
seriesId: integer("series_id").references((): any => series.id, { onDelete: "set null" }),
rating: integer("rating"), // 0..5
watched: integer("watched", { mode: "boolean" }).notNull().default(false),
isVip: integer("is_vip", { mode: "boolean" }).notNull().default(false),
isFavorite: integer("is_favorite", { mode: "boolean" }).notNull().default(false),
isOwned: integer("is_owned", { mode: "boolean" }).notNull().default(false),
hasVideo: integer("has_video", { mode: "boolean" }).notNull().default(false),
hasSubtitle: integer("has_subtitle", { mode: "boolean" }).notNull().default(false),
/** Source video codec, populated by lazy ffprobe on first playback. */
videoCodec: text("video_codec"),
/** Source video has_b_frames count (0/1/2+) — used by the auto-predicate
* playback mode to decide whether transcoding is needed. */
videoBFrames: integer("video_b_frames"),
/** Cached playback decision from auto-runtime mode: 'direct' / 'transcode'.
* Null means "not yet measured". */
playbackMode: text("playback_mode"),
notes: text("notes"),
}, (t) => ({
byCreated: index("images_created_idx").on(t.createdAt),
byDeleted: index("images_deleted_idx").on(t.deletedAt),
byParent: index("images_parent_idx").on(t.parentImageId),
byCode: index("images_code_idx").on(t.code),
byStudio: index("images_studio_idx").on(t.studioId),
byLabel: index("images_label_idx").on(t.labelId),
bySeries: index("images_series_idx").on(t.seriesId),
byHasVideo: index("images_has_video_idx").on(t.hasVideo),
byHasSubtitle: index("images_has_subtitle_idx").on(t.hasSubtitle),
}));
export const videoMetadata = sqliteTable("video_metadata", {
absPath: text("abs_path").primaryKey(),
relPath: text("rel_path").notNull(),
code: text("code").notNull(),
sizeBytes: integer("size_bytes").notNull(),
mtimeMs: real("mtime_ms").notNull(),
probedAt: integer("probed_at"),
probeError: text("probe_error"),
durationSec: real("duration_sec"),
videoCodec: text("video_codec"),
videoBFrames: integer("video_b_frames"),
width: integer("width"),
height: integer("height"),
videoBitrate: integer("video_bitrate"),
playbackMode: text("playback_mode"),
/** Suffix-pattern classification: "part" (sequential), "variant" (alt
* encode of the same content), or "single" (lone file). */
partKind: text("part_kind"),
/** Sort key for parts: 1, 2, ... — null for variants and singles. */
partIndex: integer("part_index"),
/** Stem with matched suffix stripped — variants of the same part share
* this group key. Null for singles. */
variantGroup: text("variant_group"),
}, (t) => ({
byCode: index("video_metadata_code_idx").on(t.code),
}));
export const studios = sqliteTable("studios", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull().unique(),
slug: text("slug").notNull().unique(),
notes: text("notes"),
});
export const labels = sqliteTable("labels", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull().unique(),
slug: text("slug").notNull().unique(),
notes: text("notes"),
});
export const series = sqliteTable("series", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull().unique(),
slug: text("slug").notNull().unique(),
notes: text("notes"),
});
export const actresses = sqliteTable("actresses", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull().unique(),
slug: text("slug").notNull().unique(),
altNames: text("alt_names"),
notes: text("notes"),
portraitPath: text("portrait_path"),
portraitZoom: real("portrait_zoom").notNull().default(1),
portraitOffsetX: real("portrait_offset_x").notNull().default(0),
portraitOffsetY: real("portrait_offset_y").notNull().default(0),
});
export const genres = sqliteTable("genres", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull().unique(),
slug: text("slug").notNull().unique(),
});
export const imageActresses = sqliteTable("image_actresses", {
imageId: integer("image_id").notNull().references(() => images.id, { onDelete: "cascade" }),
actressId: integer("actress_id").notNull().references(() => actresses.id, { onDelete: "cascade" }),
}, (t) => ({
pk: primaryKey({ columns: [t.imageId, t.actressId] }),
byActress: index("image_actresses_actress_idx").on(t.actressId),
}));
export const imageGenres = sqliteTable("image_genres", {
imageId: integer("image_id").notNull().references(() => images.id, { onDelete: "cascade" }),
genreId: integer("genre_id").notNull().references(() => genres.id, { onDelete: "cascade" }),
}, (t) => ({
pk: primaryKey({ columns: [t.imageId, t.genreId] }),
byGenre: index("image_genres_genre_idx").on(t.genreId),
}));
export const tagCategories = sqliteTable("tag_categories", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull().unique(),
slug: text("slug").notNull().unique(),
color: text("color"),
description: text("description"),
coverPortraitPath: text("cover_portrait_path"),
coverPortraitZoom: real("cover_portrait_zoom").notNull().default(1),
coverPortraitOffsetX: real("cover_portrait_offset_x").notNull().default(0),
coverPortraitOffsetY: real("cover_portrait_offset_y").notNull().default(0),
coverLandscapePath: text("cover_landscape_path"),
coverLandscapeZoom: real("cover_landscape_zoom").notNull().default(1),
coverLandscapeOffsetX: real("cover_landscape_offset_x").notNull().default(0),
coverLandscapeOffsetY: real("cover_landscape_offset_y").notNull().default(0),
});
export const tags = sqliteTable("tags", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull().unique(),
color: text("color"),
categoryId: integer("category_id").references((): any => tagCategories.id, { onDelete: "set null" }),
lastUsedAt: integer("last_used_at").notNull().default(0),
}, (t) => ({
byCategory: index("tags_category_idx").on(t.categoryId),
}));
export const imageTags = sqliteTable("image_tags", {
imageId: integer("image_id").notNull().references(() => images.id, { onDelete: "cascade" }),
tagId: integer("tag_id").notNull().references(() => tags.id, { onDelete: "cascade" }),
}, (t) => ({
pk: primaryKey({ columns: [t.imageId, t.tagId] }),
byTag: index("image_tags_tag_idx").on(t.tagId),
}));
export const collections = sqliteTable("collections", {
id: integer("id").primaryKey({ autoIncrement: true }),
name: text("name").notNull(),
slug: text("slug").notNull().unique(),
description: text("description"),
coverImageId: integer("cover_image_id").references(() => images.id, { onDelete: "set null" }),
createdAt: integer("created_at").notNull().default(sql`(unixepoch() * 1000)`),
position: integer("position").notNull().default(0),
lastUsedAt: integer("last_used_at").notNull().default(0),
coverPortraitPath: text("cover_portrait_path"),
coverPortraitZoom: real("cover_portrait_zoom").notNull().default(1),
coverPortraitOffsetX: real("cover_portrait_offset_x").notNull().default(0),
coverPortraitOffsetY: real("cover_portrait_offset_y").notNull().default(0),
coverLandscapePath: text("cover_landscape_path"),
coverLandscapeZoom: real("cover_landscape_zoom").notNull().default(1),
coverLandscapeOffsetX: real("cover_landscape_offset_x").notNull().default(0),
coverLandscapeOffsetY: real("cover_landscape_offset_y").notNull().default(0),
});
export const collectionImages = sqliteTable("collection_images", {
collectionId: integer("collection_id").notNull().references(() => collections.id, { onDelete: "cascade" }),
imageId: integer("image_id").notNull().references(() => images.id, { onDelete: "cascade" }),
position: integer("position").notNull().default(0),
}, (t) => ({
pk: primaryKey({ columns: [t.collectionId, t.imageId] }),
}));
export const imagesRelations = relations(images, ({ one, many }) => ({
parent: one(images, { fields: [images.parentImageId], references: [images.id], relationName: "parent" }),
studio: one(studios, { fields: [images.studioId], references: [studios.id] }),
label: one(labels, { fields: [images.labelId], references: [labels.id] }),
series: one(series, { fields: [images.seriesId], references: [series.id] }),
imageActresses: many(imageActresses),
imageGenres: many(imageGenres),
imageTags: many(imageTags),
collectionImages: many(collectionImages),
}));
export const studiosRelations = relations(studios, ({ many }) => ({ images: many(images) }));
export const labelsRelations = relations(labels, ({ many }) => ({ images: many(images) }));
export const seriesRelations = relations(series, ({ many }) => ({ images: many(images) }));
export const actressesRelations = relations(actresses, ({ many }) => ({ imageActresses: many(imageActresses) }));
export const genresRelations = relations(genres, ({ many }) => ({ imageGenres: many(imageGenres) }));
export const tagsRelations = relations(tags, ({ many }) => ({ imageTags: many(imageTags) }));
export const imageActressesRelations = relations(imageActresses, ({ one }) => ({
image: one(images, { fields: [imageActresses.imageId], references: [images.id] }),
actress: one(actresses, { fields: [imageActresses.actressId], references: [actresses.id] }),
}));
export const imageGenresRelations = relations(imageGenres, ({ one }) => ({
image: one(images, { fields: [imageGenres.imageId], references: [images.id] }),
genre: one(genres, { fields: [imageGenres.genreId], references: [genres.id] }),
}));
export const imageTagsRelations = relations(imageTags, ({ one }) => ({
image: one(images, { fields: [imageTags.imageId], references: [images.id] }),
tag: one(tags, { fields: [imageTags.tagId], references: [tags.id] }),
}));
export const collectionsRelations = relations(collections, ({ many, one }) => ({
collectionImages: many(collectionImages),
cover: one(images, { fields: [collections.coverImageId], references: [images.id] }),
}));
export const collectionImagesRelations = relations(collectionImages, ({ one }) => ({
collection: one(collections, { fields: [collectionImages.collectionId], references: [collections.id] }),
image: one(images, { fields: [collectionImages.imageId], references: [images.id] }),
}));
export type Image = typeof images.$inferSelect;
export type Studio = typeof studios.$inferSelect;
export type Label = typeof labels.$inferSelect;
export type Series = typeof series.$inferSelect;
export type Actress = typeof actresses.$inferSelect;
export type Genre = typeof genres.$inferSelect;
export type Tag = typeof tags.$inferSelect;
export type Collection = typeof collections.$inferSelect;
export type VideoMetadata = typeof videoMetadata.$inferSelect;
+102
View File
@@ -0,0 +1,102 @@
import path from "node:path";
import fs from "node:fs/promises";
const MAX_BASE_LEN = 120;
/**
* Sanitize a filename for cross-platform safety while preserving most user intent
* (unicode, spaces, dashes). Strips characters illegal on Windows + control chars,
* trims trailing dots/spaces, falls back to "image" if everything was stripped.
*/
export function sanitizeFilename(name: string): { base: string; ext: string } {
const ext = (path.extname(name) || "").toLowerCase();
let base = path.basename(name, ext);
base = base
// disallowed on Windows + path separators
.replace(/[<>:"/\\|?*\x00-\x1f]/g, "")
// collapse whitespace runs
.replace(/\s+/g, " ")
// trim trailing dots/spaces (Windows refuses these)
.replace(/[.\s]+$/g, "")
.trim();
if (base.length > MAX_BASE_LEN) base = base.slice(0, MAX_BASE_LEN).trim();
if (!base) base = "image";
return { base, ext: ext || ".png" };
}
/**
* Find a path under `dirAbs` that matches `${base}${ext}` (or `${base}-N${ext}` on
* collision) and atomically reserves it by creating a 0-byte file with the
* exclusive-create flag. The caller is expected to overwrite this placeholder
* with the real bytes; this avoids a check-then-write race where two
* concurrent uploads pick the same "unique" path.
*/
export async function uniqueFilePath(
dirAbs: string,
base: string,
ext: string,
): Promise<string> {
const tryPath = (n: number) =>
path.join(dirAbs, n === 1 ? `${base}${ext}` : `${base}-${n}${ext}`);
for (let i = 1; i < 10_000; i++) {
const p = tryPath(i);
try {
const handle = await fs.open(p, "wx");
await handle.close();
return p;
} catch (e) {
// EEXIST → another caller (or prior run) holds this name; try the
// next suffix. Anything else (perms, ENOENT on dir) is fatal.
if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
}
}
throw new Error(`Could not find unique filename for ${base}${ext} after 10000 tries`);
}
export function dayPartition(d: Date): string {
const y = d.getFullYear();
const m = String(d.getMonth() + 1).padStart(2, "0");
const day = String(d.getDate()).padStart(2, "0");
return path.posix.join(String(y), m, day);
}
/**
* Letter-bucketed storage path keyed off the first letter of a JAV code.
* Codes whose first letter isn't A-Z (null code, digit-prefix, non-Latin)
* fall into the `#` fallback bucket so the on-disk layout stays clean.
*/
const LETTER_RANGES: ReadonlyArray<{ range: string; letters: string }> = [
{ range: "A-E", letters: "ABCDE" },
{ range: "F-J", letters: "FGHIJ" },
{ range: "K-P", letters: "KLMNOP" },
{ range: "Q-U", letters: "QRSTU" },
{ range: "V-Z", letters: "VWXYZ" },
];
/**
* Canonical filename for a cover's grid-preview WebP. Embeds the JAV code
* (when known) so the data/thumbs/ folder is browsable by hand:
* "DDT-203-2aa9...945f.webp" instead of just "2aa9...945f.webp".
* Codes are validated against the same alphabet that codeParser produces
* (uppercase letters/digits/dash); anything else is dropped to keep
* filenames safe across NTFS/ext4.
*/
export function canonicalThumbName(code: string | null | undefined, sha: string): string {
const safeCode = (code ?? "").trim().toUpperCase().replace(/[^A-Z0-9-]/g, "");
return safeCode ? `${safeCode}-${sha}.webp` : `${sha}.webp`;
}
export function letterBucket(code: string | null | undefined): { range: string; letter: string; dirRel: string } {
const ch = (code ?? "").trim().charAt(0).toUpperCase();
if (!/^[A-Z]$/.test(ch)) {
// Fallback bucket is a single level — there's only one possible
// "letter" inside `#` so an extra `#/#/` layer would be redundant.
return { range: "#", letter: "#", dirRel: "#" };
}
const r = LETTER_RANGES.find((x) => x.letters.includes(ch));
if (!r) return { range: "#", letter: "#", dirRel: "#" };
return { range: r.range, letter: ch, dirRel: path.posix.join(r.range, ch) };
}
+205
View File
@@ -0,0 +1,205 @@
/**
* Shared types + URL helpers for the multi-criteria filter bar.
*/
export type FilterTabKey = "actresses" | "studios" | "series" | "genres" | "collections" | "tags" | "categories";
export type FilterMode = "or" | "and";
export type WatchedState = "all" | "watched" | "unwatched";
export type RatedState = "all" | "rated" | "unrated";
export type PresenceState = "all" | "has" | "missing";
export interface FilterStatus {
watched: WatchedState;
rated: RatedState;
collection: PresenceState;
tags: PresenceState;
video: PresenceState;
}
export type MarkState = "all" | "vip" | "favorite" | "owned" | "unmarked";
export type MarkOption = "vip" | "favorite" | "owned" | "unmarked";
export const MARK_OPTIONS: MarkOption[] = ["vip", "favorite", "owned", "unmarked"];
export interface FilterCriteria {
ids: Record<FilterTabKey, number[]>;
mode: Record<FilterTabKey, FilterMode>;
status: FilterStatus;
// Empty array == "ALL" (no mark filter). Otherwise the OR of the listed marks.
marks: MarkOption[];
}
export const EMPTY_STATUS: FilterStatus = {
watched: "all",
rated: "all",
collection: "all",
tags: "all",
video: "all",
};
export const EMPTY_CRITERIA: FilterCriteria = {
ids: { actresses: [], studios: [], series: [], genres: [], collections: [], tags: [], categories: [] },
mode: { actresses: "or", studios: "or", series: "or", genres: "or", collections: "or", tags: "or", categories: "or" },
status: EMPTY_STATUS,
marks: [],
};
export type StatusAxisKey = keyof FilterStatus;
const TAB_KEYS: FilterTabKey[] = ["actresses", "studios", "series", "genres", "collections", "tags", "categories"];
// Studios and series can't AND (a cover has at most one).
const SUPPORTS_AND: Record<FilterTabKey, boolean> = {
actresses: true,
studios: false,
series: false,
genres: true,
collections: true,
tags: true,
categories: true,
};
export function tabSupportsAnd(k: FilterTabKey): boolean {
return SUPPORTS_AND[k];
}
export function parseFilterCriteria(sp: Record<string, string | string[] | undefined>): FilterCriteria {
const c: FilterCriteria = {
ids: { actresses: [], studios: [], series: [], genres: [], collections: [], tags: [], categories: [] },
mode: { actresses: "or", studios: "or", series: "or", genres: "or", collections: "or", tags: "or", categories: "or" },
status: { ...EMPTY_STATUS },
marks: [],
};
for (const k of TAB_KEYS) {
const raw = sp[k];
if (typeof raw === "string" && raw.trim()) {
c.ids[k] = raw.split(",").map((x) => Number(x)).filter((n) => Number.isFinite(n) && n > 0);
}
const m = sp[`${k}_mode`];
if (typeof m === "string" && m.toLowerCase() === "and" && SUPPORTS_AND[k]) c.mode[k] = "and";
}
// Watched axis. Accept both new (?watched=watched|unwatched) and legacy (?watched=1, ?unwatched=1).
const watched = sp.watched;
if (typeof watched === "string") {
if (watched === "watched" || watched === "1") c.status.watched = "watched";
else if (watched === "unwatched") c.status.watched = "unwatched";
}
if (sp.unwatched === "1") c.status.watched = "unwatched";
const rated = sp.rated;
if (typeof rated === "string") {
if (rated === "rated" || rated === "1") c.status.rated = "rated";
else if (rated === "unrated") c.status.rated = "unrated";
}
if (sp.unrated === "1") c.status.rated = "unrated";
const collection = sp.collection;
if (typeof collection === "string") {
if (collection === "has") c.status.collection = "has";
else if (collection === "missing") c.status.collection = "missing";
}
if (sp.uncollected === "1") c.status.collection = "missing";
const tagsStatus = sp.tags_status;
if (typeof tagsStatus === "string") {
if (tagsStatus === "has") c.status.tags = "has";
else if (tagsStatus === "missing") c.status.tags = "missing";
}
if (sp.untagged === "1") c.status.tags = "missing";
const video = sp.video;
if (typeof video === "string") {
if (video === "has") c.status.video = "has";
else if (video === "missing") c.status.video = "missing";
}
// marks: comma-separated list. Legacy `?mark=vip` (singular) is also honored
// so old bookmarked URLs keep working.
const marksRaw = (typeof sp.marks === "string" && sp.marks) || (typeof sp.mark === "string" ? sp.mark : "");
if (marksRaw) {
const wanted = marksRaw.split(",").map((s) => s.trim()).filter(Boolean);
const valid = new Set<MarkOption>(MARK_OPTIONS);
const out: MarkOption[] = [];
const seen = new Set<MarkOption>();
for (const w of wanted) {
if (valid.has(w as MarkOption) && !seen.has(w as MarkOption)) {
out.push(w as MarkOption);
seen.add(w as MarkOption);
}
}
c.marks = out;
}
return c;
}
/** Apply a `FilterCriteria` onto a `URLSearchParams`. Existing keys are overwritten. */
export function writeFilterCriteria(sp: URLSearchParams, c: FilterCriteria): void {
for (const k of TAB_KEYS) {
if (c.ids[k].length === 0) sp.delete(k);
else sp.set(k, c.ids[k].join(","));
if (c.mode[k] === "and" && SUPPORTS_AND[k]) sp.set(`${k}_mode`, "and");
else sp.delete(`${k}_mode`);
}
// Always write canonical status keys; clear the legacy ones so they don't fight us.
if (c.status.watched === "all") sp.delete("watched"); else sp.set("watched", c.status.watched);
if (c.status.rated === "all") sp.delete("rated"); else sp.set("rated", c.status.rated);
if (c.status.collection === "all") sp.delete("collection"); else sp.set("collection", c.status.collection);
if (c.status.tags === "all") sp.delete("tags_status"); else sp.set("tags_status", c.status.tags);
if (c.status.video === "all") sp.delete("video"); else sp.set("video", c.status.video);
sp.delete("unwatched");
sp.delete("unrated");
sp.delete("uncollected");
sp.delete("untagged");
// Single canonical key: `marks=vip,favorite`. Drop legacy `mark` if present.
sp.delete("mark");
if (c.marks.length === 0) sp.delete("marks");
else sp.set("marks", c.marks.join(","));
}
export function totalSelected(c: FilterCriteria): number {
let n = 0;
for (const k of TAB_KEYS) n += c.ids[k].length;
return n;
}
export function totalStatusActive(c: FilterCriteria): number {
let n = 0;
if (c.status.watched !== "all") n++;
if (c.status.rated !== "all") n++;
if (c.status.collection !== "all") n++;
if (c.status.tags !== "all") n++;
if (c.status.video !== "all") n++;
return n;
}
export function anyActive(c: FilterCriteria): boolean {
return totalSelected(c) > 0 || totalStatusActive(c) > 0 || c.marks.length > 0;
}
/** Translate the tri-state status into the boolean flags consumed by listImages / libraryLetterCounts. */
export function statusToFlags(s: FilterStatus): {
watched?: boolean;
unwatched?: boolean;
rated?: boolean;
unrated?: boolean;
hasCollection?: boolean;
uncollected?: boolean;
hasTags?: boolean;
untagged?: boolean;
hasVideo?: boolean;
noVideo?: boolean;
} {
return {
watched: s.watched === "watched",
unwatched: s.watched === "unwatched",
rated: s.rated === "rated",
unrated: s.rated === "unrated",
hasCollection: s.collection === "has",
uncollected: s.collection === "missing",
hasTags: s.tags === "has",
untagged: s.tags === "missing",
hasVideo: s.video === "has",
noVideo: s.video === "missing",
};
}
+16
View File
@@ -0,0 +1,16 @@
import { useEffect, type RefObject } from "react";
export function useClickOutside<T extends HTMLElement>(
ref: RefObject<T | null>,
onOutside: () => void,
enabled = true,
) {
useEffect(() => {
if (!enabled) return;
const handler = (e: MouseEvent) => {
if (ref.current && !ref.current.contains(e.target as Node)) onOutside();
};
document.addEventListener("mousedown", handler);
return () => document.removeEventListener("mousedown", handler);
}, [ref, onOutside, enabled]);
}
+389
View File
@@ -0,0 +1,389 @@
import "server-only";
import path from "node:path";
import fs from "node:fs/promises";
import crypto from "node:crypto";
import sharp from "sharp";
import { db, rawDb } from "@/lib/db/client";
import { images } from "@/lib/db/schema";
import { eq } from "drizzle-orm";
import { sanitizeFilename, uniqueFilePath, letterBucket, canonicalThumbName } from "@/lib/filename";
import { extractCode, normalizeCode } from "@/lib/jav/codeParser";
import { computeDHash } from "@/lib/jav/phash";
import { parseNfo, type NfoMetadata } from "@/lib/jav/nfoParser";
import { upsertStudio, upsertSeries, upsertActress, upsertGenre } from "@/lib/jav/upsert";
const LIBRARY_ROOT = path.join(process.cwd(), "library");
const THUMB_ROOT = path.join(process.cwd(), "data", "thumbs");
const SUPERSEDED_ROOT = path.join(process.cwd(), "library", ".superseded");
export type CollisionBucket = "upgrade" | "downgrade" | "sidegrade" | "mixed";
export interface CollisionInfo {
existingId: number;
existingFilename: string;
existingWidth: number;
existingHeight: number;
existingBytes: number;
existingThumbPath: string;
incomingWidth: number;
incomingHeight: number;
incomingBytes: number;
bucket: CollisionBucket;
}
export interface IngestResult {
imageId: number;
duplicate: boolean;
filename: string;
code: string | null;
/** Present when the upload was deferred because a row with the same
* canonical code already exists. The caller must re-invoke ingest with
* resolution: "replace" or "skip". When "skip", the staged file has
* already been cleaned up and no DB write happens. */
collision?: CollisionInfo;
}
function classifyCollision(
oldW: number, oldH: number, oldBytes: number,
newW: number, newH: number, newBytes: number,
): CollisionBucket {
const oldPx = oldW * oldH;
const newPx = newW * newH;
// Upgrade: incoming has ≥1.5× pixel area.
if (newPx >= oldPx * 1.5) return "upgrade";
// Downgrade: incoming smaller in both dims AND bytes.
if (newW <= oldW && newH <= oldH && newBytes <= oldBytes && (newW < oldW || newH < oldH || newBytes < oldBytes)) {
return "downgrade";
}
// Sidegrade: dims within ±2px and bytes within ±15%.
const dimsClose = Math.abs(newW - oldW) <= 2 && Math.abs(newH - oldH) <= 2;
const bytesClose = Math.abs(newBytes - oldBytes) <= oldBytes * 0.15;
if (dimsClose && bytesClose) return "sidegrade";
return "mixed";
}
export async function ingestFile(
buffer: Buffer,
originalFilename: string,
opts?: {
/** Optional .nfo XML payload to seed metadata from. */
nfoXml?: string;
autoAssign?: { tagName?: string; collectionId?: number };
/** When set, the new image is attached as an extra (back cover / still) of this parent. */
parentImageId?: number;
/** Override filename to store on disk and in the DB (e.g. "DDK-134.jpg"). */
targetFilename?: string;
/** Explicit actress names to attach (link existing or create-new). */
actressNames?: string[];
/** When set, controls how a same-code collision (different SHA) is
* resolved. "detect" (default) returns a collision result without
* writing. "replace" overwrites the existing row's bytes/sha/dims
* in place, preserving relational state. "skip" returns the existing
* row unchanged. */
onCollision?: "detect" | "replace" | "skip";
},
): Promise<IngestResult> {
const sha = crypto.createHash("sha256").update(buffer).digest("hex");
const existing = db.select().from(images).where(eq(images.sha256, sha)).get();
if (existing) {
// If we're re-uploading an existing attachment, re-bind that attachment to
// the requested parent. Do not turn an existing cover into its own child.
if (opts?.parentImageId != null) {
if (existing.parentImageId != null && existing.id !== opts.parentImageId) {
rawDb.prepare(`
UPDATE images
SET parent_image_id = ?, deleted_at = NULL
WHERE id = ?
`).run(opts.parentImageId, existing.id);
}
} else if (existing.deletedAt != null) {
// Plain re-upload of a soft-deleted cover: revive it.
rawDb.prepare(`UPDATE images SET deleted_at = NULL WHERE id = ?`).run(existing.id);
}
// Re-uploads can carry fresh actress decisions from the preview
// dialog. Merge them into the existing row's links so duplicates
// aren't a dead end for metadata. INSERT OR IGNORE keeps already-
// linked actresses as no-ops; only attach to top-level covers.
if (opts?.actressNames?.length && existing.parentImageId == null) {
for (const name of opts.actressNames) {
const trimmed = name.trim();
if (!trimmed) continue;
const id = upsertActress(trimmed);
rawDb.prepare(`INSERT OR IGNORE INTO image_actresses (image_id, actress_id) VALUES (?, ?)`).run(existing.id, id);
}
}
if (opts?.autoAssign) applyAutoAssign(existing.id, opts.autoAssign);
return { imageId: existing.id, duplicate: true, filename: existing.filename, code: existing.code };
}
const filenameForStorage = opts?.targetFilename?.trim() || originalFilename;
const { base, ext } = sanitizeFilename(filenameForStorage);
// Resolve metadata BEFORE choosing the bucket: the on-disk partition is
// keyed off the cover's first letter, so we need the code (or the
// parent's code, for attached images) up front.
const isAttached = opts?.parentImageId != null;
if (isAttached) {
const parent = rawDb.prepare(`
SELECT id FROM images
WHERE id = ? AND deleted_at IS NULL AND parent_image_id IS NULL
`).get(opts.parentImageId) as { id: number } | undefined;
if (!parent) throw new Error("Attachment parent not found");
}
const nfo = opts?.nfoXml ? parseNfo(opts.nfoXml) : null;
const code = isAttached
? null
: (normalizeCode(nfo?.code) ?? extractCode(filenameForStorage) ?? extractCode(originalFilename));
let bucketCode: string | null = code;
if (isAttached) {
const parentRow = rawDb.prepare(`SELECT code FROM images WHERE id = ?`).get(opts.parentImageId) as
| { code: string | null }
| undefined;
bucketCode = parentRow?.code ?? null;
}
const dirRel = letterBucket(bucketCode).dirRel;
const dirAbs = path.join(LIBRARY_ROOT, dirRel);
await fs.mkdir(dirAbs, { recursive: true });
const fileAbs = await uniqueFilePath(dirAbs, base, ext);
const fileRel = path.posix.join(dirRel, path.basename(fileAbs));
await fs.mkdir(THUMB_ROOT, { recursive: true });
// Use the bucket code (which already accounts for attached → parent's
// code) as the prefix so attached thumbs sort with their cover.
const thumbName = canonicalThumbName(isAttached ? bucketCode : code, sha);
const thumbAbs = path.join(THUMB_ROOT, thumbName);
// If thumb generation or metadata extraction fails, clean up the source
// file we just wrote — otherwise it's an orphan in library/ with no DB row.
let width = 0;
let height = 0;
let phash: string | null = null;
try {
await fs.writeFile(fileAbs, buffer);
const meta = await sharp(buffer, { failOn: "none" }).metadata();
width = meta.width ?? 0;
height = meta.height ?? 0;
await sharp(buffer, { failOn: "none" })
.rotate()
.resize({ width: 768, height: 768, fit: "inside", withoutEnlargement: true })
.webp({ quality: 82 })
.toFile(thumbAbs);
// Perceptual hash for near-duplicate detection. Failure here is
// non-fatal — we just leave phash null and the maintenance scanner
// can backfill later.
try { phash = await computeDHash(buffer); } catch { phash = null; }
} catch (e) {
await fs.rm(fileAbs, { force: true }).catch(() => {});
await fs.rm(thumbAbs, { force: true }).catch(() => {});
throw e;
}
// Collision detection: a primary cover (no parent, not soft-deleted) with
// the same canonical code already exists. We've already missed SHA dedup,
// so this is a different encode of the same release.
if (!isAttached && code) {
const collision = rawDb.prepare(`
SELECT id, filename, rel_path, thumb_path, width, height, bytes
FROM images
WHERE code = ? AND parent_image_id IS NULL AND deleted_at IS NULL
ORDER BY id LIMIT 1
`).get(code) as
| { id: number; filename: string; rel_path: string; thumb_path: string; width: number; height: number; bytes: number }
| undefined;
if (collision) {
const mode = opts?.onCollision ?? "detect";
if (mode === "skip") {
await fs.rm(fileAbs, { force: true }).catch(() => {});
await fs.rm(thumbAbs, { force: true }).catch(() => {});
return { imageId: collision.id, duplicate: true, filename: collision.filename, code };
}
if (mode === "replace") {
// Move the old file + thumb to .superseded/ for recovery, then
// update the existing row in place. All relational state
// (actresses, tags, collections, rating, watched, notes) is
// preserved because we keep the same row id.
await fs.mkdir(SUPERSEDED_ROOT, { recursive: true });
const stamp = Date.now();
const oldExt = path.extname(collision.rel_path) || ".bin";
const supersededFile = path.join(SUPERSEDED_ROOT, `${collision.id}-${stamp}${oldExt}`);
const supersededThumb = path.join(SUPERSEDED_ROOT, `${collision.id}-${stamp}.thumb.webp`);
try {
await fs.rename(path.join(LIBRARY_ROOT, collision.rel_path), supersededFile).catch(() => {});
await fs.rename(path.join(THUMB_ROOT, collision.thumb_path), supersededThumb).catch(() => {});
} catch {
// Best-effort recovery copy; proceed even if the old files
// were already missing on disk.
}
const update = rawDb.transaction(() => {
rawDb.prepare(`
UPDATE images SET
filename = ?, rel_path = ?, thumb_path = ?, sha256 = ?,
width = ?, height = ?, bytes = ?, phash = ?
WHERE id = ?
`).run(filenameForStorage, fileRel, thumbName, sha, width, height, buffer.length, phash, collision.id);
});
try {
update();
} catch (e) {
// Restore on failure (e.g. UNIQUE(sha256) clash with an unrelated row).
await fs.rename(supersededFile, path.join(LIBRARY_ROOT, collision.rel_path)).catch(() => {});
await fs.rename(supersededThumb, path.join(THUMB_ROOT, collision.thumb_path)).catch(() => {});
await fs.rm(fileAbs, { force: true }).catch(() => {});
await fs.rm(thumbAbs, { force: true }).catch(() => {});
throw e;
}
// Replace upgrades bytes but should also merge any fresh actress
// decisions the user made — same semantics as the dedup branch
// up top. Existing actress links are preserved; INSERT OR IGNORE
// only adds new ones.
if (opts?.actressNames?.length) {
for (const name of opts.actressNames) {
const trimmed = name.trim();
if (!trimmed) continue;
const id = upsertActress(trimmed);
rawDb.prepare(`INSERT OR IGNORE INTO image_actresses (image_id, actress_id) VALUES (?, ?)`).run(collision.id, id);
}
}
if (opts?.autoAssign) applyAutoAssign(collision.id, opts.autoAssign);
return { imageId: collision.id, duplicate: false, filename: filenameForStorage, code };
}
// mode === "detect": back out the staged files, return collision
// info, and let the caller decide. No DB write.
await fs.rm(fileAbs, { force: true }).catch(() => {});
await fs.rm(thumbAbs, { force: true }).catch(() => {});
return {
imageId: collision.id,
duplicate: false,
filename: filenameForStorage,
code,
collision: {
existingId: collision.id,
existingFilename: collision.filename,
existingWidth: collision.width,
existingHeight: collision.height,
existingBytes: collision.bytes,
existingThumbPath: collision.thumb_path,
incomingWidth: width,
incomingHeight: height,
incomingBytes: buffer.length,
bucket: classifyCollision(
collision.width, collision.height, collision.bytes,
width, height, buffer.length,
),
},
};
}
}
const studioId = !isAttached && nfo?.studio ? upsertStudio(nfo.studio) : null;
const seriesId = !isAttached && nfo?.series ? upsertSeries(nfo.series) : null;
const insert = rawDb.transaction(() => {
const result = rawDb.prepare(`
INSERT INTO images (
filename, rel_path, thumb_path, sha256, width, height, bytes,
parent_image_id, code, title, release_date, runtime_min, director,
studio_id, series_id, notes, phash
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
filenameForStorage,
fileRel,
thumbName,
sha,
width,
height,
buffer.length,
opts?.parentImageId ?? null,
code,
isAttached ? null : (nfo?.title ?? null),
isAttached ? null : (nfo?.releaseDate ?? null),
isAttached ? null : (nfo?.runtimeMin ?? null),
isAttached ? null : (nfo?.director ?? null),
studioId,
seriesId,
isAttached ? null : (nfo?.notes ?? null),
phash,
);
const imageId = Number(result.lastInsertRowid);
if (!isAttached && nfo) {
attachNfoChildren(imageId, nfo);
}
if (!isAttached && opts?.actressNames && opts.actressNames.length > 0) {
for (const name of opts.actressNames) {
const trimmed = name.trim();
if (!trimmed) continue;
const id = upsertActress(trimmed);
rawDb.prepare(`INSERT OR IGNORE INTO image_actresses (image_id, actress_id) VALUES (?, ?)`).run(imageId, id);
}
}
return imageId;
});
let imageId: number;
try {
imageId = insert();
} catch (e) {
// Concurrent uploads of the same file can race past the dedup check
// above; the UNIQUE(sha256) / UNIQUE(rel_path) constraints will catch
// the loser. Treat as duplicate and clean up the file we just wrote.
const msg = (e as Error).message ?? "";
if (/UNIQUE constraint failed/i.test(msg)) {
await fs.rm(fileAbs, { force: true }).catch(() => {});
await fs.rm(thumbAbs, { force: true }).catch(() => {});
const winner = db.select().from(images).where(eq(images.sha256, sha)).get();
if (winner) {
if (opts?.autoAssign) applyAutoAssign(winner.id, opts.autoAssign);
return { imageId: winner.id, duplicate: true, filename: winner.filename, code: winner.code };
}
}
throw e;
}
if (opts?.autoAssign) applyAutoAssign(imageId, opts.autoAssign);
return { imageId, duplicate: false, filename: filenameForStorage, code };
}
function attachNfoChildren(imageId: number, nfo: NfoMetadata) {
if (nfo.actresses) {
for (const name of nfo.actresses) {
const id = upsertActress(name);
rawDb.prepare(`INSERT OR IGNORE INTO image_actresses (image_id, actress_id) VALUES (?, ?)`).run(imageId, id);
}
}
if (nfo.genres) {
for (const name of nfo.genres) {
const id = upsertGenre(name);
rawDb.prepare(`INSERT OR IGNORE INTO image_genres (image_id, genre_id) VALUES (?, ?)`).run(imageId, id);
}
}
}
function applyAutoAssign(imageId: number, opts: { tagName?: string; collectionId?: number }) {
if (opts.tagName) {
const trimmed = opts.tagName.trim().toLowerCase();
if (trimmed) {
const tag = rawDb.prepare(`
INSERT INTO tags (name) VALUES (?) ON CONFLICT(name) DO UPDATE SET name=excluded.name RETURNING id
`).get(trimmed) as { id: number };
rawDb.prepare(`INSERT OR IGNORE INTO image_tags (image_id, tag_id) VALUES (?, ?)`).run(imageId, tag.id);
}
}
if (opts.collectionId != null) {
const collectionId = opts.collectionId;
const tx = rawDb.transaction(() => {
const max = rawDb.prepare(`SELECT COALESCE(MAX(position), -1) AS m FROM collection_images WHERE collection_id = ?`).get(collectionId) as { m: number };
rawDb.prepare(`
INSERT OR IGNORE INTO collection_images (collection_id, image_id, position) VALUES (?, ?, ?)
`).run(collectionId, imageId, max.m + 1);
});
tx();
}
}
+50
View File
@@ -0,0 +1,50 @@
/**
* Extract a JAV product code from a filename.
* Handles common patterns: "SSIS-001.jpg", "[ABC] DEF-123.png", "ssis001 cover.jpg",
* "abp-456_cover.jpeg", and pre-fixes like "FHD-MIDV-123.jpg".
*
* Returns the canonical "PREFIX-NUMBER" form (uppercased), or null if no match.
*/
// Prefix must start with a letter and may contain digits (e.g. "T28", "ABP",
// "MIAA"). Total prefix length 2-6 alphanumerics. The numeric tail is 3-5
// digits, with an optional single-letter suffix for variant releases —
// "IBW-610z" (z-version / re-cut), "ABP-456a" (part-a), etc. Suffix is
// preserved in the canonical form so e.g. IBW-610 and IBW-610Z stay
// distinct (they're different versions of the same release).
const CODE_RE = /\b([A-Za-z][A-Za-z0-9]{1,5})[-_ ]?(\d{3,5})([a-zA-Z])?\b/;
const QUALITY_PREFIXES = new Set(["fhd", "hd", "sd", "uhd", "4k", "1080p", "720p", "480p", "hevc", "x265", "x264"]);
/**
* Canonicalise an already-known code to "PREFIX-NUMBER" with the number
* left-padded to a minimum of 3 digits and the prefix uppercased. Inputs
* coming from NFO `<id>` tags can arrive in any of "MIAA-291", "miaa291",
* "MIAA-00291", etc.; we want them all to compare equal so the collision
* detector isn't fooled by formatting drift.
*/
export function normalizeCode(raw: string | null | undefined): string | null {
if (!raw) return null;
const m = raw.trim().match(CODE_RE);
if (!m) return null;
const suffix = (m[3] ?? "").toUpperCase();
return `${m[1].toUpperCase()}-${m[2].padStart(3, "0")}${suffix}`;
}
export function extractCode(filename: string): string | null {
const base = filename.replace(/\.[^.]+$/, "");
// Walk left-to-right, skipping quality prefixes that look like codes (e.g. "FHD-").
let cursor = 0;
while (cursor < base.length) {
const m = base.slice(cursor).match(CODE_RE);
if (!m) return null;
const prefix = m[1];
const num = m[2];
const suffix = (m[3] ?? "").toUpperCase();
if (QUALITY_PREFIXES.has(prefix.toLowerCase())) {
cursor += (m.index ?? 0) + m[0].length;
continue;
}
return `${prefix.toUpperCase()}-${num.padStart(3, "0")}${suffix}`;
}
return null;
}
+43
View File
@@ -0,0 +1,43 @@
import { extractCode } from "./codeParser";
export interface ParsedDropFilename {
/** Original filename with extension. */
original: string;
/** Canonical "PREFIX-NUMBER" code, or null. */
code: string | null;
/** Actress display names parsed from filename (after the first " - "). */
actresses: string[];
/** Suggested filename to save as (CODE.ext) when code is known; otherwise the original. */
targetFilename: string;
}
/**
* Parse a dropped image filename of the form:
* "DDK-134.jpg"
* "DDK-134 - Shuri Atomi.jpg"
* "DDK-134 - Shuri Atomi, Reika Aiba.jpg"
*
* The first " - " (space-hyphen-space) after the code separates the code/title from
* the actress list. Multiple actresses are comma-separated.
*/
export function parseDroppedFilename(name: string): ParsedDropFilename {
const extMatch = name.match(/\.([^.]+)$/);
const ext = extMatch ? extMatch[1] : "";
const stem = ext ? name.slice(0, -(ext.length + 1)) : name;
const code = extractCode(name);
let actresses: string[] = [];
const sepIdx = stem.indexOf(" - ");
if (sepIdx !== -1) {
const tail = stem.slice(sepIdx + 3).trim();
if (tail) {
actresses = tail
.split(/\s*,\s*/)
.map((s) => s.trim())
.filter(Boolean);
}
}
const targetFilename = code ? (ext ? `${code}.${ext}` : code) : name;
return { original: name, code, actresses, targetFilename };
}
+93
View File
@@ -0,0 +1,93 @@
import { parseNfo, type NfoMetadata } from "./nfoParser";
/**
* Lenient JSON metadata parser. Accepts a wide range of common keys produced by
* scrapers (Kodi/Jellyfin/Stash/JavSP/etc.) and normalizes them into the same
* shape as `NfoMetadata` so the cover editor can pre-fill from a file or a
* pasted blob.
*/
export function parseMetaJson(raw: string): NfoMetadata | null {
let obj: unknown;
try {
obj = JSON.parse(raw);
} catch {
return null;
}
if (!obj || typeof obj !== "object") return null;
// Some scrapers wrap in {"data": {...}} or arrays.
const root = (obj as Record<string, unknown>);
const data = (root.data && typeof root.data === "object" ? root.data : root) as Record<string, unknown>;
const pick = (...keys: string[]): string | undefined => {
for (const k of keys) {
const v = data[k];
if (typeof v === "string" && v.trim()) return v.trim();
if (typeof v === "number" && Number.isFinite(v)) return String(v);
}
return undefined;
};
const pickArray = (...keys: string[]): string[] => {
for (const k of keys) {
const v = data[k];
if (Array.isArray(v)) {
return v
.map((x) => {
if (typeof x === "string") return x.trim();
if (x && typeof x === "object") {
const name = (x as Record<string, unknown>).name ?? (x as Record<string, unknown>).Name;
return typeof name === "string" ? name.trim() : "";
}
return "";
})
.filter(Boolean);
}
if (typeof v === "string" && v.trim()) {
return v.split(/[,、,]/).map((s) => s.trim()).filter(Boolean);
}
}
return [];
};
const runtimeRaw = pick("runtime", "runtimeMin", "runtime_min", "duration");
const runtimeMin = runtimeRaw ? parseInt(runtimeRaw, 10) : NaN;
const out: NfoMetadata = {
title: pick("title", "originaltitle", "originalTitle", "name"),
code: pick("code", "id", "num", "number", "javId", "jav_id"),
releaseDate: pick("releaseDate", "release_date", "premiered", "releasedate", "date", "year"),
runtimeMin: Number.isFinite(runtimeMin) ? runtimeMin : undefined,
director: pick("director"),
studio: pick("studio", "maker", "manufacturer"),
series: pick("series", "set"),
actresses: pickArray("actresses", "actors", "actor", "performers", "cast", "stars"),
genres: pickArray("genres", "genre", "tags", "categories"),
notes: pick("plot", "outline", "summary", "description", "notes"),
};
return strip(out);
}
/**
* Try JSON first, fall back to XML. Returns null if neither matches.
*/
export function parseMetaAny(raw: string): NfoMetadata | null {
const trimmed = raw.trim();
if (!trimmed) return null;
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
return parseMetaJson(trimmed);
}
return parseNfo(trimmed);
}
function strip(o: NfoMetadata): NfoMetadata {
const out: NfoMetadata = {};
for (const [k, v] of Object.entries(o)) {
if (v == null) continue;
if (Array.isArray(v) && v.length === 0) continue;
if (typeof v === "string" && v === "") continue;
(out as Record<string, unknown>)[k] = v;
}
return out;
}
+38
View File
@@ -0,0 +1,38 @@
/**
* Reverse the word order of a name. "Aiba Reika" → "Reika Aiba".
* For single-token names returns null (no useful reverse).
*/
export function reverseName(name: string): string | null {
const tokens = name.trim().split(/\s+/).filter(Boolean);
if (tokens.length < 2) return null;
return tokens.slice().reverse().join(" ");
}
/**
* Build the displayed alt-name list for an actress: always lead with the
* word-order-reversed name (auto-generated, user can't remove), then any
* user-typed alt names from the comma-separated `altNames` field, deduped
* and excluding the actress's own canonical name.
*/
export function buildAltNameChips(name: string, altNamesField: string | null): Array<{ value: string; auto: boolean }> {
const out: Array<{ value: string; auto: boolean }> = [];
const seen = new Set<string>([name.trim().toLowerCase()]);
const reversed = reverseName(name);
if (reversed && !seen.has(reversed.toLowerCase())) {
out.push({ value: reversed, auto: true });
seen.add(reversed.toLowerCase());
}
if (altNamesField) {
for (const raw of altNamesField.split(/[,、,]/)) {
const v = raw.trim();
if (!v) continue;
const k = v.toLowerCase();
if (seen.has(k)) continue;
seen.add(k);
out.push({ value: v, auto: false });
}
}
return out;
}
+85
View File
@@ -0,0 +1,85 @@
/**
* Minimal Kodi/Jellyfin-style .nfo parser. Operates on a string that contains
* an XML document and extracts the fields we map onto cover metadata.
*
* No XML library dependency — small, regex-based, deliberately lenient.
* Returns null if the buffer doesn't look like a movie .nfo.
*/
export interface NfoMetadata {
title?: string;
code?: string;
releaseDate?: string;
runtimeMin?: number;
director?: string;
studio?: string;
series?: string;
actresses?: string[];
genres?: string[];
notes?: string;
}
const TAG_RE = (tag: string) =>
new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)<\\/${tag}>`, "gi");
function firstMatch(xml: string, tag: string): string | undefined {
const re = TAG_RE(tag);
const m = re.exec(xml);
if (!m) return undefined;
return decodeEntities(m[1].trim());
}
function allMatches(xml: string, tag: string): string[] {
const re = TAG_RE(tag);
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = re.exec(xml)) !== null) {
const text = decodeEntities(m[1].trim());
if (text) out.push(text);
}
return out;
}
function decodeEntities(s: string): string {
return s
.replace(/<!\[CDATA\[([\s\S]*?)]]>/g, "$1")
.replace(/&amp;/g, "&")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'");
}
export function parseNfo(xml: string): NfoMetadata | null {
if (!/<movie\b|<id\b|<title\b/i.test(xml)) return null;
const runtimeRaw = firstMatch(xml, "runtime");
const runtime = runtimeRaw ? parseInt(runtimeRaw, 10) : NaN;
return strip({
title: firstMatch(xml, "title") ?? firstMatch(xml, "originaltitle"),
code: firstMatch(xml, "id") ?? firstMatch(xml, "num"),
releaseDate: firstMatch(xml, "premiered") ?? firstMatch(xml, "releasedate") ?? firstMatch(xml, "year"),
runtimeMin: Number.isFinite(runtime) ? runtime : undefined,
director: firstMatch(xml, "director"),
studio: firstMatch(xml, "studio") ?? firstMatch(xml, "maker"),
series: firstMatch(xml, "set") ?? firstMatch(xml, "series"),
actresses: allMatches(xml, "actor").map((block) => {
const name = block.match(/<name>([\s\S]*?)<\/name>/i);
return decodeEntities(name ? name[1].trim() : block.trim());
}).filter(Boolean),
genres: [...allMatches(xml, "genre"), ...allMatches(xml, "tag")],
notes: firstMatch(xml, "plot") ?? firstMatch(xml, "outline"),
});
}
function strip(o: NfoMetadata): NfoMetadata {
const out: NfoMetadata = {};
for (const [k, v] of Object.entries(o)) {
if (v == null) continue;
if (Array.isArray(v) && v.length === 0) continue;
if (typeof v === "string" && v === "") continue;
(out as Record<string, unknown>)[k] = v;
}
return out;
}
+55
View File
@@ -0,0 +1,55 @@
import sharp from "sharp";
/**
* Difference-hash (dHash) implementation. Resizes the input to 9×8 grayscale
* and emits one bit per adjacent-pixel comparison: if the left pixel is
* brighter than the right, bit = 1. Result is 64 bits, encoded as a 16-char
* lowercase hex string.
*
* dHash is robust to scaling, mild JPEG/WebP recompression, brightness
* tweaks, and small crops — exactly the cases the SHA-256 dedup misses
* (same picture, different bytes). Hamming-distance ≤ ~10 between two
* hashes is a strong "same image, different encode" signal in practice.
*/
export async function computeDHash(input: Buffer): Promise<string> {
// 9 wide × 8 tall, grayscale, raw pixel bytes.
const buf = await sharp(input, { failOn: "none" })
.rotate()
.grayscale()
.resize(9, 8, { fit: "fill" })
.raw()
.toBuffer();
let hex = "";
for (let row = 0; row < 8; row++) {
let byte = 0;
for (let col = 0; col < 8; col++) {
const left = buf[row * 9 + col];
const right = buf[row * 9 + col + 1];
if (left > right) byte |= 1 << (7 - col);
}
hex += byte.toString(16).padStart(2, "0");
}
return hex;
}
/**
* Hamming distance between two 16-char hex strings. Returns Infinity if
* the inputs aren't both well-formed 64-bit hashes.
*/
export function hammingDistance(a: string, b: string): number {
if (a.length !== 16 || b.length !== 16) return Infinity;
let dist = 0;
for (let i = 0; i < 16; i += 2) {
const byteA = parseInt(a.slice(i, i + 2), 16);
const byteB = parseInt(b.slice(i, i + 2), 16);
if (!Number.isFinite(byteA) || !Number.isFinite(byteB)) return Infinity;
let xor = byteA ^ byteB;
// popcount on a single byte
while (xor) {
dist += xor & 1;
xor >>>= 1;
}
}
return dist;
}
+18
View File
@@ -0,0 +1,18 @@
import "server-only";
import { rawDb, uniqueSlug } from "@/lib/db/client";
function upsert(table: "studios" | "labels" | "series" | "actresses" | "genres", name: string): number {
const trimmed = name.trim();
if (!trimmed) throw new Error(`empty ${table.slice(0, -1)} name`);
const existing = rawDb.prepare(`SELECT id FROM ${table} WHERE name = ?`).get(trimmed) as { id: number } | undefined;
if (existing) return existing.id;
const slug = uniqueSlug(rawDb, table, trimmed);
const row = rawDb.prepare(`INSERT INTO ${table} (name, slug) VALUES (?, ?) RETURNING id`).get(trimmed, slug) as { id: number };
return row.id;
}
export const upsertStudio = (name: string) => upsert("studios", name);
export const upsertLabel = (name: string) => upsert("labels", name);
export const upsertSeries = (name: string) => upsert("series", name);
export const upsertActress = (name: string) => upsert("actresses", name);
export const upsertGenre = (name: string) => upsert("genres", name);
+58
View File
@@ -0,0 +1,58 @@
import "server-only";
import path from "node:path";
import fs from "node:fs/promises";
/**
* Resolve `relPath` against `root` and confirm the result stays inside
* `root`. Returns null on any escape attempt or invalid input.
*
* Defense layers:
* - Reject empty/null/non-string input.
* - Reject NUL bytes (POSIX truncation tricks).
* - Reject absolute paths (drive letters, leading slash). path.resolve
* silently honors absolute second args, which would let a caller
* escape root before path.relative even runs.
* - Reject Windows-style separators on POSIX so `..\foo` isn't smuggled
* past the relative-check.
* - Final containment check via path.relative — escape paths surface
* as `..` or remain absolute.
*
* Symlinks are NOT resolved (fs is async, callers are sync). Use
* safeRealJoin for paths that may contain user-controlled symlinks.
*/
export function safeJoin(root: string, relPath: string | null | undefined): string | null {
if (!relPath || typeof relPath !== "string") return null;
if (relPath.includes("\0")) return null;
if (path.isAbsolute(relPath)) return null;
// path.win32.isAbsolute treats `C:` and `\\server\share` as absolute;
// on POSIX path.isAbsolute won't catch those, so check explicitly.
if (process.platform !== "win32" && /^[a-zA-Z]:[\\/]|^\\\\/.test(relPath)) return null;
const abs = path.resolve(root, relPath);
const rel = path.relative(root, abs);
if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) return null;
return abs;
}
/**
* Async variant: like safeJoin, but additionally resolves symlinks via
* realpath and re-verifies the resolved target stays inside root.
* Use when the path may live under a directory the user can write
* symlinks into (e.g. external library/subtitle roots).
*/
export async function safeRealJoin(root: string, relPath: string | null | undefined): Promise<string | null> {
const abs = safeJoin(root, relPath);
if (!abs) return null;
let real: string;
try {
real = await fs.realpath(abs);
} catch {
// File may not exist yet (e.g. about-to-be-written target). Fall
// back to the static check — caller is responsible for not creating
// symlinks at this path.
return abs;
}
const rootReal = await fs.realpath(root).catch(() => path.resolve(root));
const rel = path.relative(rootReal, real);
if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) return null;
return real;
}
+24
View File
@@ -0,0 +1,24 @@
import { BRAND } from "./brand";
export type SortKey = "newest" | "oldest" | "az" | "za" | "code-az" | "code-za";
export const SORT_OPTIONS: ReadonlyArray<{ value: SortKey; label: string }> = [
{ value: "newest", label: "Newest First" },
{ value: "oldest", label: "Oldest First" },
{ value: "az", label: "Title A → Z" },
{ value: "za", label: "Title Z → A" },
{ value: "code-az", label: "Code A → Z" },
{ value: "code-za", label: "Code Z → A" },
];
export const DEFAULT_SORT: SortKey = "newest";
export const SORT_COOKIE = `${BRAND.storagePrefix}-default-sort`;
const VALID = new Set<SortKey>(["newest", "oldest", "az", "za", "code-az", "code-za"]);
export function isValidSort(v: string | undefined | null): v is SortKey {
return !!v && VALID.has(v as SortKey);
}
export function labelFor(sort: SortKey): string {
return SORT_OPTIONS.find((o) => o.value === sort)?.label ?? sort;
}
+22
View File
@@ -0,0 +1,22 @@
import "server-only";
import { cookies } from "next/headers";
import { isValidSort, SORT_COOKIE, type SortKey } from "./sort";
import { getAppSetting } from "./db/appSettings";
/**
* Resolution order: explicit URL ?sort= > server-persisted setting > cookie > default.
* The cookie is fallback only (e.g. before any setting is saved); the DB value
* survives anything short of deleting the database.
*/
export async function resolveSort(urlSort?: string): Promise<SortKey> {
if (isValidSort(urlSort)) return urlSort;
const stored = getAppSetting("defaultSort");
if (isValidSort(stored)) return stored;
const cookie = (await cookies()).get(SORT_COOKIE)?.value;
if (isValidSort(cookie)) return cookie;
return "newest";
}
export function getStoredDefaultSort(): SortKey {
return getAppSetting("defaultSort");
}
+35
View File
@@ -0,0 +1,35 @@
import { clsx, type ClassValue } from "clsx";
import { twMerge } from "tailwind-merge";
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
}
export function formatBytes(n: number): string {
if (n < 1024) return `${n} B`;
const u = ["KB", "MB", "GB"];
let v = n / 1024;
let i = 0;
while (v >= 1024 && i < u.length - 1) { v /= 1024; i++; }
return `${v.toFixed(v < 10 ? 1 : 0)} ${u[i]}`;
}
export function coverHref(image: { id: number; code: string | null }): string {
if (image.code && image.code.trim()) return `/id/${encodeURIComponent(image.code)}`;
return `/image/${image.id}`;
}
export function relativeTime(ms: number): string {
const diff = Date.now() - ms;
const s = Math.floor(diff / 1000);
if (s < 60) return "just now";
const m = Math.floor(s / 60);
if (m < 60) return `${m}m ago`;
const h = Math.floor(m / 60);
if (h < 24) return `${h}h ago`;
const d = Math.floor(h / 24);
if (d < 30) return `${d}d ago`;
const mo = Math.floor(d / 30);
if (mo < 12) return `${mo}mo ago`;
return `${Math.floor(mo / 12)}y ago`;
}
+59
View File
@@ -0,0 +1,59 @@
import "server-only";
import { spawn } from "node:child_process";
const cache = new Map<string, number>();
const PROBE_TIMEOUT_MS = 10_000;
/**
* Probe a video file's duration in seconds via ffprobe. Cached per-path
* for the lifetime of the process — files don't change duration on us.
* Returns null if ffprobe fails or returns garbage.
*
* Caps the probe at PROBE_TIMEOUT_MS and ties to an optional AbortSignal
* so a hung ffprobe (network mount, weird codec, dead disk) can't leave
* the request awaiting forever or zombie the subprocess.
*/
export async function probeDuration(abs: string, signal?: AbortSignal): Promise<number | null> {
const cached = cache.get(abs);
if (cached !== undefined) return cached;
return new Promise((resolve) => {
const proc = spawn("ffprobe", [
"-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
abs,
]);
let out = "";
let settled = false;
const settle = (n: number | null) => {
if (settled) return;
settled = true;
if (timeoutId) clearTimeout(timeoutId);
if (signal && abortHandler) signal.removeEventListener("abort", abortHandler);
if (n != null && Number.isFinite(n) && n > 0) {
cache.set(abs, n);
resolve(n);
} else {
resolve(null);
}
};
const kill = () => {
try { proc.kill("SIGKILL"); } catch { /* ignore */ }
settle(null);
};
const timeoutId = setTimeout(kill, PROBE_TIMEOUT_MS);
const abortHandler = signal ? () => kill() : null;
if (signal && abortHandler) {
if (signal.aborted) { kill(); return; }
signal.addEventListener("abort", abortHandler, { once: true });
}
proc.stdout?.on("data", (d) => { out += d.toString(); });
proc.on("close", () => settle(Number(out.trim())));
proc.on("error", () => settle(null));
});
}
+539
View File
@@ -0,0 +1,539 @@
import "server-only";
import path from "node:path";
import fs from "node:fs/promises";
import { extractCode, normalizeCode } from "@/lib/jav/codeParser";
import { getAppSetting } from "@/lib/db/appSettings";
import { rawDb } from "@/lib/db/client";
import { syncVideoMetadataIndex } from "./metadata";
export const VIDEO_EXTENSIONS = new Set([
".mp4", ".mkv", ".m4v", ".mov", ".webm", ".avi", ".wmv", ".ts", ".mpg", ".mpeg", ".flv",
]);
const SUBTITLE_EXTENSIONS = new Set([".srt", ".vtt", ".ass", ".ssa"]);
/** One video file the index found on disk. */
export interface VideoFile {
/** Absolute path on disk. */
abs: string;
/** Path relative to the configured video library root. */
rel: string;
/** Filename (with extension). */
filename: string;
/** Normalized JAV code parsed from the filename. */
code: string;
/** File size in bytes. */
size: number;
/** Last-modified timestamp (ms). */
mtime: number;
}
/**
* Lightweight scan-state record. Authoritative file data lives in the
* `video_metadata` SQLite table — accessors below query it directly,
* so this struct holds only what describes the most recent rescan.
*/
interface VideoIndex {
/** When the index was last built. */
lastScannedAt: number;
/** All folder roots that were scanned, in order: main first, extras after.
* Used both to display in the UI and to detect setting changes. */
rootsScanned: string[];
/** Total files matched by the most recent scan. */
count: number;
}
const EMPTY_INDEX: VideoIndex = {
lastScannedAt: 0,
rootsScanned: [],
count: 0,
};
let cachedScanState: VideoIndex = EMPTY_INDEX;
let scanInFlight: Promise<VideoIndex> | null = null;
interface CachedFileRow {
abs_path: string;
rel_path: string;
code: string;
size_bytes: number;
mtime_ms: number;
}
interface WalkOpts {
/** When true, ignore the dir-mtime cache and re-readdir every dir.
* Use after structural file edits that don't change dir mtime
* (e.g. content rewrite without rename). */
force?: boolean;
}
/**
* Walk the configured roots and produce a flat VideoFile[]. The caller
* writes the result to the `video_metadata` table — nothing is held in
* memory beyond the duration of one rescan.
*
* Incremental: each directory's mtime is compared to a stored value
* in `video_dir_mtimes`. If unchanged, the immediate-children file
* rows for that dir are reused from `video_metadata` instead of
* readdir + stat per file. Subdirs are still walked (their mtimes
* may have changed independently).
*/
async function walkAllRoots(
roots: string[],
opts: WalkOpts = {},
): Promise<{ files: VideoFile[]; count: number; visitedDirs: Set<string>; reused: number; rescanned: number }> {
const cachedMtimes = opts.force
? new Map<string, number>()
: loadDirMtimeCache();
const visitedDirs = new Set<string>();
const files: VideoFile[] = [];
const cachedFilesByDir = opts.force
? new Map<string, CachedFileRow[]>()
: loadCachedFileIndex();
let reused = 0;
let rescanned = 0;
for (const root of roots) {
type Frame = { dir: string };
const stack: Frame[] = [{ dir: root }];
while (stack.length) {
const { dir } = stack.pop()!;
visitedDirs.add(dir);
let dirStat: import("node:fs").Stats;
try {
dirStat = await fs.stat(dir);
} catch {
continue; // dir vanished mid-walk
}
const cachedMtime = cachedMtimes.get(dir);
const dirUnchanged = cachedMtime != null && cachedMtime === dirStat.mtimeMs;
// Always recurse — subdir mtimes are tracked independently.
// For *children* enumeration we use cached rows when unchanged.
// We still need the subdir list either way; if we're skipping
// the readdir for cache reuse, we need an alternate way to find
// subdirs. Cheapest: readdir the directory entries once for
// dirs (tiny per-dir cost) and use the dirent type directly.
let entries: import("node:fs").Dirent[];
try {
entries = await fs.readdir(dir, { withFileTypes: true });
} catch {
continue;
}
// Push subdirs onto the stack regardless of cache state.
for (const e of entries) {
if (e.isDirectory()) {
stack.push({ dir: path.join(dir, e.name) });
}
}
if (dirUnchanged) {
// Reuse cached rows for files immediately in this directory.
const cached = cachedFilesByDir.get(dir);
if (cached) {
for (const row of cached) {
files.push({
abs: row.abs_path,
rel: path.relative(root, row.abs_path),
filename: path.basename(row.abs_path),
code: row.code,
size: row.size_bytes,
mtime: row.mtime_ms,
});
}
reused += cached.length;
}
continue;
}
// Dir changed (or no cache entry yet). Readdir + stat each file.
rescanned++;
for (const e of entries) {
if (!e.isFile()) continue;
const ext = path.extname(e.name).toLowerCase();
if (!VIDEO_EXTENSIONS.has(ext)) continue;
const abs = path.join(dir, e.name);
const stem = e.name.slice(0, e.name.length - ext.length);
const code = extractCode(stem);
if (!code) continue;
const norm = normalizeCode(code);
if (!norm) continue;
let st: import("node:fs").Stats;
try {
st = await fs.stat(abs);
} catch {
continue;
}
files.push({
abs,
rel: path.relative(root, abs),
filename: e.name,
code: norm,
size: st.size,
mtime: st.mtimeMs,
});
}
// Update cached mtime so the NEXT scan sees this dir as fresh.
cachedMtimes.set(dir, dirStat.mtimeMs);
}
}
// Persist updated mtime cache for next scan.
saveDirMtimeCache(cachedMtimes, visitedDirs);
// Stable order across rescans.
files.sort((a, b) => a.code.localeCompare(b.code) || a.filename.localeCompare(b.filename));
return { files, count: files.length, visitedDirs, reused, rescanned };
}
/** Load all `video_dir_mtimes` rows into a Map keyed by abs_dir. */
function loadDirMtimeCache(): Map<string, number> {
const rows = rawDb.prepare(`SELECT abs_dir, mtime_ms FROM video_dir_mtimes`).all() as Array<{ abs_dir: string; mtime_ms: number }>;
const out = new Map<string, number>();
for (const r of rows) out.set(r.abs_dir, r.mtime_ms);
return out;
}
/** Group the entire video_metadata table by dir_path so dir-cache
* reuse is a single in-memory lookup per dir. One linear scan of the
* table — cheap even at 80k rows. */
function loadCachedFileIndex(): Map<string, CachedFileRow[]> {
const rows = rawDb.prepare(`
SELECT abs_path, rel_path, code, size_bytes, mtime_ms, dir_path
FROM video_metadata
`).all() as Array<CachedFileRow & { dir_path: string }>;
const out = new Map<string, CachedFileRow[]>();
for (const r of rows) {
const arr = out.get(r.dir_path);
if (arr) arr.push(r);
else out.set(r.dir_path, [r]);
}
return out;
}
/** Upsert dir mtimes for visited dirs and prune rows for dirs we
* didn't see this scan (deleted folders). */
function saveDirMtimeCache(mtimes: Map<string, number>, visited: Set<string>): void {
const upsert = rawDb.prepare(`
INSERT INTO video_dir_mtimes (abs_dir, mtime_ms, last_seen_at)
VALUES (?, ?, ?)
ON CONFLICT(abs_dir) DO UPDATE SET
mtime_ms = excluded.mtime_ms,
last_seen_at = excluded.last_seen_at
`);
const now = Date.now();
const tx = rawDb.transaction(() => {
for (const [dir, mtime] of mtimes) {
// Only persist dirs we actually visited this scan — others may
// have been moved/renamed and their cache entry is stale.
if (!visited.has(dir)) continue;
upsert.run(dir, mtime, now);
}
// Prune rows whose dir we didn't see this scan. Drops cleanup of
// deleted dirs in O(rows) — fine at any reasonable scale.
const allRows = rawDb.prepare(`SELECT abs_dir FROM video_dir_mtimes`).all() as Array<{ abs_dir: string }>;
const del = rawDb.prepare(`DELETE FROM video_dir_mtimes WHERE abs_dir = ?`);
for (const r of allRows) {
if (!visited.has(r.abs_dir)) del.run(r.abs_dir);
}
});
try {
tx();
} catch (e) {
console.error("[video] failed to save dir mtime cache:", e);
}
}
/**
* Walk every place a sidecar subtitle could live and return the set of
* canonical codes that have at least one. Cheap signal — no ffprobe.
*
* - Each video's own directory, filtered to filenames that start with
* the video stem (so a stray `OTHER-001.srt` next to `YUJ-001.mp4`
* doesn't taint YUJ-001).
* - Each entry in `subtitleExtraPaths` (recursive walk, depth 3) —
* extracts the code from the filename directly.
* - data/generated-subtitles/<code>/ — directory name IS the code.
*
* Result is consumed once by syncHasSubtitleColumn and discarded — no
* persistent in-memory copy.
*/
async function collectSubtitleCodes(files: VideoFile[]): Promise<Set<string>> {
const codes = new Set<string>();
// Same-folder scan: per video, look at sibling files. Cache directory
// listings so a folder with N videos is only listed once.
const dirCache = new Map<string, import("node:fs").Dirent[]>();
for (const file of files) {
const dir = path.dirname(file.abs);
let entries = dirCache.get(dir);
if (!entries) {
try {
entries = await fs.readdir(dir, { withFileTypes: true });
} catch {
entries = [];
}
dirCache.set(dir, entries);
}
const stem = file.filename.slice(0, file.filename.length - path.extname(file.filename).length);
const stemLower = stem.toLowerCase();
const codeLower = file.code.toLowerCase();
for (const e of entries) {
if (!e.isFile()) continue;
const ext = path.extname(e.name).toLowerCase();
if (!SUBTITLE_EXTENSIONS.has(ext)) continue;
const lower = e.name.toLowerCase();
// Code-substring match must treat the code as a delimited token
// (start, end, or wrapped in non-alphanumeric) — bare `.includes`
// would attribute `notes-yuj-001-bad.srt` to YUJ-001.
const codeAsToken = (() => {
const idx = lower.indexOf(codeLower);
if (idx < 0) return false;
const before = idx === 0 ? "" : lower[idx - 1]!;
const afterIdx = idx + codeLower.length;
const after = afterIdx >= lower.length ? "" : lower[afterIdx]!;
const isBoundary = (c: string) => c === "" || !/[a-z0-9]/.test(c);
return isBoundary(before) && isBoundary(after);
})();
if (lower.startsWith(stemLower + ".") || lower === stemLower + ext || codeAsToken) {
codes.add(file.code);
break;
}
}
}
// Persistent subtitle library roots — extract codes from filenames.
const extraRoots = (getAppSetting("subtitleExtraPaths") ?? []).filter(Boolean);
for (const root of extraRoots) {
await walkSubtitleRoot(root, codes, 3);
}
// data/generated-subtitles/<code>/ — directory name is the code.
const generatedRoot = path.join(process.cwd(), "data", "generated-subtitles");
try {
const subdirs = await fs.readdir(generatedRoot, { withFileTypes: true });
for (const d of subdirs) {
if (!d.isDirectory()) continue;
const dirAbs = path.join(generatedRoot, d.name);
let entries: import("node:fs").Dirent[];
try {
entries = await fs.readdir(dirAbs, { withFileTypes: true });
} catch {
continue;
}
const hasSub = entries.some(
(e) => e.isFile() && SUBTITLE_EXTENSIONS.has(path.extname(e.name).toLowerCase()),
);
if (hasSub) {
const norm = normalizeCode(d.name);
if (norm) codes.add(norm);
}
}
} catch { /* generated-subtitles not present yet — fine */ }
return codes;
}
async function walkSubtitleRoot(root: string, out: Set<string>, maxDepth: number): Promise<void> {
type Frame = { dir: string; depth: number };
const stack: Frame[] = [{ dir: root, depth: 0 }];
while (stack.length) {
const { dir, depth } = stack.pop()!;
let entries: import("node:fs").Dirent[];
try {
entries = await fs.readdir(dir, { withFileTypes: true });
} catch {
continue;
}
for (const e of entries) {
const full = path.join(dir, e.name);
if (e.isDirectory()) {
if (depth < maxDepth) stack.push({ dir: full, depth: depth + 1 });
} else if (e.isFile()) {
const ext = path.extname(e.name).toLowerCase();
if (!SUBTITLE_EXTENSIONS.has(ext)) continue;
const stem = e.name.slice(0, e.name.length - ext.length);
const code = extractCode(stem);
if (!code) continue;
const norm = normalizeCode(code);
if (norm) out.add(norm);
}
}
}
}
/** Exposed for path-allowlist checks (e.g. subtitle file resolution). */
export function getConfiguredVideoRoots(): string[] {
return configuredRoots();
}
function configuredRoots(): string[] {
const main = (getAppSetting("videoLibraryPath") || "").trim();
const extras = getAppSetting("videoExtraPaths") ?? [];
const out: string[] = [];
if (main) out.push(main);
for (const e of extras) {
const t = (e ?? "").trim();
if (t) out.push(t);
}
return out;
}
function rootsEqual(a: string[], b: string[]): boolean {
if (a.length !== b.length) return false;
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
return true;
}
/**
* Scan-state probe — used by API routes to decide whether the cached
* data still matches current settings. Returns the empty state if the
* configured roots have changed (caller can trigger a rescan).
*/
export function getVideoIndex(): VideoIndex {
const roots = configuredRoots();
if (roots.length === 0) return EMPTY_INDEX;
if (!rootsEqual(cachedScanState.rootsScanned, roots)) return EMPTY_INDEX;
return cachedScanState;
}
/** Rebuild the index from disk. Coalesces concurrent calls. Authoritative
* data lands in the `video_metadata` table; this function returns only
* scan-state metadata.
*
* Default mode is incremental — directories whose mtime hasn't
* changed since the last scan reuse cached file rows without
* readdir-per-file. Pass `{force:true}` to bypass the dir-mtime
* cache (e.g. after content edits that don't bump dir mtime). */
export async function rescanVideoIndex(opts: { force?: boolean } = {}): Promise<VideoIndex> {
const roots = configuredRoots();
if (scanInFlight) return scanInFlight;
scanInFlight = (async () => {
try {
const cleanRoots = roots.map((r) => (r ?? "").trim()).filter(Boolean);
if (cleanRoots.length === 0) {
cachedScanState = { ...EMPTY_INDEX };
return cachedScanState;
}
const t0 = Date.now();
const { files, count, reused, rescanned } = await walkAllRoots(cleanRoots, { force: opts.force });
const walkMs = Date.now() - t0;
console.log(
`[video] rescan walk in ${walkMs}ms — ${count} files (${reused} reused, ${rescanned} dir(s) rewalked${opts.force ? ", forced" : ""})`,
);
// Persist the file table first — has_video / has_subtitle bulk
// updates and metadata sync all run off it.
await syncVideoMetadataIndex(files);
syncHasVideoColumn(files);
const subtitleCodes = await collectSubtitleCodes(files);
syncHasSubtitleColumn(subtitleCodes);
cachedScanState = {
lastScannedAt: Date.now(),
rootsScanned: cleanRoots,
count,
};
return cachedScanState;
} finally {
scanInFlight = null;
}
})();
return scanInFlight;
}
/**
* Mirror the freshly-walked code list into images.has_video so SQL
* filters / counts can use the column directly.
*/
function syncHasVideoColumn(files: VideoFile[]): void {
const codes = Array.from(new Set(files.map((f) => f.code)));
const tx = rawDb.transaction(() => {
rawDb.prepare(`UPDATE images SET has_video = 0 WHERE has_video = 1`).run();
if (codes.length === 0) return;
// Chunk to stay well below SQLite's bind-parameter cap.
const CHUNK = 500;
for (let i = 0; i < codes.length; i += CHUNK) {
const slice = codes.slice(i, i + CHUNK);
const placeholders = slice.map(() => "?").join(",");
rawDb.prepare(
`UPDATE images SET has_video = 1 WHERE upper(code) IN (${placeholders})`,
).run(...slice);
}
});
try {
tx();
} catch (e) {
console.error("[video] failed to sync has_video column:", e);
}
}
/** Mirror the freshly-walked subtitle code set into images.has_subtitle. */
function syncHasSubtitleColumn(subtitleCodes: Set<string>): void {
const codes = Array.from(subtitleCodes);
const tx = rawDb.transaction(() => {
rawDb.prepare(`UPDATE images SET has_subtitle = 0 WHERE has_subtitle = 1`).run();
if (codes.length === 0) return;
const CHUNK = 500;
for (let i = 0; i < codes.length; i += CHUNK) {
const slice = codes.slice(i, i + CHUNK);
const placeholders = slice.map(() => "?").join(",");
rawDb.prepare(
`UPDATE images SET has_subtitle = 1 WHERE upper(code) IN (${placeholders})`,
).run(...slice);
}
});
try {
tx();
} catch (e) {
console.error("[video] failed to sync has_subtitle column:", e);
}
}
interface VideoMetaRow {
abs_path: string;
rel_path: string;
code: string;
size_bytes: number;
mtime_ms: number;
}
/** Look up files for a single normalized code. Reads directly from the
* video_metadata table so the result is always current with the most
* recent rescan. */
export function findVideosForCode(code: string | null | undefined): VideoFile[] {
if (!code) return [];
const norm = normalizeCode(code) ?? code.toUpperCase();
const rows = rawDb.prepare(`
SELECT abs_path, rel_path, code, size_bytes, mtime_ms
FROM video_metadata
WHERE upper(code) = ?
ORDER BY rel_path COLLATE NOCASE
`).all(norm) as VideoMetaRow[];
return rows.map((r) => ({
abs: r.abs_path,
rel: r.rel_path,
filename: path.basename(r.abs_path),
code: r.code,
size: r.size_bytes,
mtime: r.mtime_ms,
}));
}
/** Set of every code present in video_metadata — fast existence check. */
export function getCodesWithVideos(): Set<string> {
const rows = rawDb.prepare(`
SELECT DISTINCT upper(code) AS code FROM video_metadata
`).all() as Array<{ code: string }>;
return new Set(rows.map((r) => r.code));
}
/** Set of every code with a discoverable subtitle sidecar. Reads from
* the images.has_subtitle column populated at rescan time. */
export function getCodesWithSubtitles(): Set<string> {
const rows = rawDb.prepare(`
SELECT DISTINCT upper(code) AS code FROM images WHERE has_subtitle = 1 AND code IS NOT NULL
`).all() as Array<{ code: string }>;
return new Set(rows.map((r) => r.code));
}
+58
View File
@@ -0,0 +1,58 @@
import "server-only";
import path from "node:path";
import { rawDb } from "@/lib/db/client";
export interface ManualSubtitle {
code: string;
partIdx: number;
absPath: string;
attachedAt: number;
}
interface ManualSubtitleRow {
code: string;
part_idx: number;
abs_path: string;
attached_at: number;
}
function rowToEntry(r: ManualSubtitleRow): ManualSubtitle {
return { code: r.code, partIdx: r.part_idx, absPath: r.abs_path, attachedAt: r.attached_at };
}
export function listManualSubtitlesForVariant(code: string, partIdx: number): ManualSubtitle[] {
const rows = rawDb.prepare(`
SELECT code, part_idx, abs_path, attached_at FROM manual_subtitles
WHERE code = ? AND part_idx = ?
ORDER BY attached_at DESC
`).all(code, partIdx) as ManualSubtitleRow[];
return rows.map(rowToEntry);
}
/** True iff this exact abs path is recorded against any (code, part). */
export function isManualSubtitlePath(abs: string): boolean {
const resolved = path.resolve(abs);
// Windows paths are case-insensitive on disk but stored as-typed.
// Compare with a case-insensitive LIKE on Windows, exact on POSIX.
if (process.platform === "win32") {
const row = rawDb.prepare(`
SELECT 1 FROM manual_subtitles WHERE LOWER(abs_path) = LOWER(?) LIMIT 1
`).get(resolved);
return !!row;
}
const row = rawDb.prepare(`SELECT 1 FROM manual_subtitles WHERE abs_path = ? LIMIT 1`).get(resolved);
return !!row;
}
export function attachManualSubtitle(code: string, partIdx: number, absPath: string): void {
rawDb.prepare(`
INSERT OR REPLACE INTO manual_subtitles (code, part_idx, abs_path, attached_at)
VALUES (?, ?, ?, ?)
`).run(code, partIdx, path.resolve(absPath), Date.now());
}
export function detachManualSubtitle(code: string, partIdx: number, absPath: string): void {
rawDb.prepare(`
DELETE FROM manual_subtitles WHERE code = ? AND part_idx = ? AND abs_path = ?
`).run(code, partIdx, path.resolve(absPath));
}
+580
View File
@@ -0,0 +1,580 @@
import "server-only";
import path from "node:path";
import { spawn } from "node:child_process";
import fs from "node:fs/promises";
import { revalidatePath } from "next/cache";
import { rawDb } from "@/lib/db/client";
import { getAppSetting } from "@/lib/db/appSettings";
import { classifyGroup, compilePatterns } from "./partClassify";
import type { VideoFile } from "./index";
const PROBE_TIMEOUT_MS = 10_000;
export type PlaybackMode = "direct" | "transcode";
export interface StoredVideoMetadata {
absPath: string;
relPath: string;
code: string;
sizeBytes: number;
mtimeMs: number;
probedAt: number | null;
probeError: string | null;
durationSec: number | null;
videoCodec: string | null;
videoBFrames: number | null;
width: number | null;
height: number | null;
videoBitrate: number | null;
playbackMode: PlaybackMode | null;
partKind: "part" | "variant" | "single" | null;
partIndex: number | null;
variantGroup: string | null;
}
interface VideoMetadataRow {
abs_path: string;
rel_path: string;
code: string;
size_bytes: number;
mtime_ms: number;
probed_at: number | null;
probe_error: string | null;
duration_sec: number | null;
video_codec: string | null;
video_b_frames: number | null;
width: number | null;
height: number | null;
video_bitrate: number | null;
playback_mode: string | null;
part_kind: string | null;
part_index: number | null;
variant_group: string | null;
}
interface FfprobeJson {
streams?: Array<{
codec_name?: string;
width?: number;
height?: number;
bit_rate?: string;
has_b_frames?: number;
}>;
format?: {
duration?: string;
bit_rate?: string;
};
}
function mapRow(row: VideoMetadataRow | undefined): StoredVideoMetadata | null {
if (!row) return null;
return {
absPath: row.abs_path,
relPath: row.rel_path,
code: row.code,
sizeBytes: row.size_bytes,
mtimeMs: row.mtime_ms,
probedAt: row.probed_at,
probeError: row.probe_error,
durationSec: row.duration_sec,
videoCodec: row.video_codec,
videoBFrames: row.video_b_frames,
width: row.width,
height: row.height,
videoBitrate: row.video_bitrate,
playbackMode: row.playback_mode === "direct" || row.playback_mode === "transcode" ? row.playback_mode : null,
partKind: row.part_kind === "part" || row.part_kind === "variant" || row.part_kind === "single" ? row.part_kind : null,
partIndex: row.part_index,
variantGroup: row.variant_group,
};
}
function parseFiniteNumber(value: unknown): number | null {
if (value == null || value === "N/A") return null;
const n = typeof value === "number" ? value : Number(value);
return Number.isFinite(n) && n > 0 ? n : null;
}
function parseNonNegativeNumber(value: unknown): number | null {
if (value == null || value === "N/A") return null;
const n = typeof value === "number" ? value : Number(value);
return Number.isFinite(n) && n >= 0 ? n : null;
}
function isStatMatch(row: StoredVideoMetadata, sizeBytes: number, mtimeMs: number): boolean {
return row.sizeBytes === sizeBytes && Math.abs(row.mtimeMs - mtimeMs) < 1;
}
export function getStoredVideoMetadata(absPath: string): StoredVideoMetadata | null {
return mapRow(rawDb.prepare(`SELECT * FROM video_metadata WHERE abs_path = ?`).get(absPath) as VideoMetadataRow | undefined);
}
export function listStoredVideoMetadataForCode(code: string | null | undefined): StoredVideoMetadata[] {
if (!code) return [];
const rows = rawDb.prepare(`
SELECT * FROM video_metadata
WHERE upper(code) = upper(?)
ORDER BY rel_path ASC
`).all(code) as VideoMetadataRow[];
return rows.map((row) => mapRow(row)).filter((row): row is StoredVideoMetadata => row !== null);
}
export function serializeVideoMetadata(meta: StoredVideoMetadata | null) {
if (!meta) return null;
return {
absPath: meta.absPath,
relPath: meta.relPath,
code: meta.code,
sizeBytes: meta.sizeBytes,
mtimeMs: meta.mtimeMs,
probedAt: meta.probedAt,
probeError: meta.probeError,
durationSec: meta.durationSec,
videoCodec: meta.videoCodec,
videoBFrames: meta.videoBFrames,
width: meta.width,
height: meta.height,
videoBitrate: meta.videoBitrate,
playbackMode: meta.playbackMode,
partKind: meta.partKind,
partIndex: meta.partIndex,
variantGroup: meta.variantGroup,
};
}
export async function syncVideoMetadataIndex(files: VideoFile[]): Promise<void> {
const found = new Set(files.map((file) => file.abs));
const upsert = rawDb.prepare(`
INSERT INTO video_metadata (abs_path, rel_path, code, size_bytes, mtime_ms, dir_path)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(abs_path) DO UPDATE SET
rel_path = excluded.rel_path,
code = excluded.code,
dir_path = excluded.dir_path,
probed_at = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.probed_at
END,
probe_error = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.probe_error
END,
duration_sec = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.duration_sec
END,
video_codec = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.video_codec
END,
video_b_frames = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.video_b_frames
END,
width = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.width
END,
height = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.height
END,
video_bitrate = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.video_bitrate
END,
playback_mode = CASE
WHEN video_metadata.size_bytes != excluded.size_bytes OR video_metadata.mtime_ms != excluded.mtime_ms THEN NULL
ELSE video_metadata.playback_mode
END,
size_bytes = excluded.size_bytes,
mtime_ms = excluded.mtime_ms
`);
const deleteStale = rawDb.prepare(`DELETE FROM video_metadata WHERE abs_path = ?`);
const tx = rawDb.transaction(() => {
for (const file of files) {
const last = Math.max(file.abs.lastIndexOf("/"), file.abs.lastIndexOf("\\"));
const dir = last >= 0 ? file.abs.slice(0, last) : "";
upsert.run(file.abs, file.rel, file.code, file.size, file.mtime, dir);
}
const rows = rawDb.prepare(`SELECT abs_path FROM video_metadata`).all() as Array<{ abs_path: string }>;
for (const row of rows) {
if (!found.has(row.abs_path)) deleteStale.run(row.abs_path);
}
});
tx();
classifyAndPersist(files);
// Probe-data refresh runs in the background. Awaiting here used to
// block rescan responses for minutes on libraries with many drifted
// files (e.g. after a bulk rename). Each per-file probe completion
// calls revalidatePath internally so detail pages update as soon as
// their own video is fresh — no batch-level waiting.
void reprobeDirtyFiles(files);
}
const REPROBE_CONCURRENCY = 2;
async function reprobeDirtyFiles(files: VideoFile[]): Promise<void> {
let dirty: Array<{ abs_path: string }>;
try {
dirty = rawDb
.prepare(`SELECT abs_path FROM video_metadata WHERE probed_at IS NULL AND probe_error IS NULL`)
.all() as Array<{ abs_path: string }>;
} catch (e) {
console.error("[video] reprobe-dirty query failed:", e);
return;
}
if (dirty.length === 0) return;
const dirtySet = new Set(dirty.map((r) => r.abs_path));
const targets = files.filter((f) => dirtySet.has(f.abs));
if (targets.length === 0) return;
// Process in chunks of REPROBE_CONCURRENCY. ffprobe is mostly waiting
// on disk; small parallelism is enough.
let cursor = 0;
const workers: Promise<void>[] = [];
// Throttle revalidation calls: a burst of 1000 path invalidations
// would itself thrash. Coalesce so each batch of N codes triggers
// one revalidate per code, deduped within a short window.
const codesSeen = new Set<string>();
for (let i = 0; i < REPROBE_CONCURRENCY; i++) {
workers.push((async () => {
while (cursor < targets.length) {
const idx = cursor++;
const file = targets[idx];
if (!file) break;
try {
await probeVideoMetadata(file);
if (!codesSeen.has(file.code)) {
codesSeen.add(file.code);
try { revalidatePath("/id/[code]", "page"); } catch { /* ignore */ }
}
} catch (e) {
console.error(`[video] reprobe failed for ${file.abs}:`, e);
}
}
})());
}
await Promise.all(workers).catch(() => { /* swallowed */ });
}
/**
* Recompute part/variant classification for every file based on the
* current `partSuffixPatterns` setting. Independent of probe data; safe
* to run on every scan.
*/
function classifyAndPersist(files: VideoFile[]): void {
const sources = getAppSetting("partSuffixPatterns") ?? [];
const patterns = compilePatterns(sources);
const byCode = new Map<string, VideoFile[]>();
for (const f of files) {
const arr = byCode.get(f.code);
if (arr) arr.push(f);
else byCode.set(f.code, [f]);
}
const update = rawDb.prepare(`
UPDATE video_metadata SET part_kind = ?, part_index = ?, variant_group = ?
WHERE abs_path = ?
`);
const tx = rawDb.transaction(() => {
for (const group of byCode.values()) {
const inputs = group.map((f) => ({
key: f.abs,
stem: stemOf(f.filename),
}));
const results = classifyGroup(inputs, patterns);
for (const r of results) {
update.run(r.partKind, r.partIndex, r.variantGroup, r.key);
}
}
});
tx();
}
function stemOf(filename: string): string {
const ext = path.extname(filename);
return ext ? filename.slice(0, -ext.length) : filename;
}
export interface SubtitleStreamInfo {
index: number;
codec: string;
language: string | null;
title: string | null;
isImageBased: boolean;
isTextBased: boolean;
}
const TEXT_SUBTITLE_CODECS = new Set(["subrip", "ass", "ssa", "mov_text", "webvtt", "text"]);
const IMAGE_SUBTITLE_CODECS = new Set(["hdmv_pgs_subtitle", "dvd_subtitle", "dvb_subtitle", "dvbsub", "pgssub"]);
interface FfprobeStream {
index?: number;
codec_type?: string;
codec_name?: string;
tags?: { language?: string; title?: string };
}
/** Enumerate subtitle streams in a container. Computed on demand — not
* persisted, since users frequently remux subs in/out and a stale list
* is worse than re-probing. Returns [] on error or missing ffprobe. */
export async function runFfprobeSubtitles(absPath: string): Promise<SubtitleStreamInfo[]> {
return new Promise((resolve) => {
const proc = spawn("ffprobe", [
"-v", "error",
"-select_streams", "s",
"-show_entries", "stream=index,codec_name,codec_type:stream_tags=language,title",
"-of", "json",
absPath,
]);
let out = "";
let settled = false;
const settle = (val: SubtitleStreamInfo[]) => { if (!settled) { settled = true; clearTimeout(t); resolve(val); } };
const t = setTimeout(() => { try { proc.kill("SIGKILL"); } catch {} settle([]); }, PROBE_TIMEOUT_MS);
proc.stdout?.on("data", (d) => { out += d.toString(); });
proc.on("error", () => settle([]));
proc.on("close", (code) => {
if (code !== 0) { settle([]); return; }
try {
const json = JSON.parse(out) as { streams?: FfprobeStream[] };
const streams = (json.streams ?? []).filter((s) => s.codec_type === "subtitle");
const result: SubtitleStreamInfo[] = streams.map((s, i) => {
const codec = (s.codec_name ?? "unknown").toLowerCase();
return {
// Use the per-codec_type ordinal — that's what ffmpeg's
// 0:s:N mapping wants, NOT the absolute stream index.
index: i,
codec,
language: typeof s.tags?.language === "string" ? s.tags.language : null,
title: typeof s.tags?.title === "string" ? s.tags.title : null,
isImageBased: IMAGE_SUBTITLE_CODECS.has(codec),
isTextBased: TEXT_SUBTITLE_CODECS.has(codec),
};
});
settle(result);
} catch {
settle([]);
}
});
});
}
async function runFfprobe(absPath: string, signal?: AbortSignal): Promise<{
durationSec: number | null;
videoCodec: string | null;
videoBFrames: number | null;
width: number | null;
height: number | null;
videoBitrate: number | null;
}> {
return new Promise((resolve, reject) => {
const proc = spawn("ffprobe", [
"-v", "error",
"-select_streams", "v:0",
"-show_entries", "stream=codec_name,width,height,bit_rate,has_b_frames:format=duration,bit_rate",
"-of", "json",
absPath,
]);
let out = "";
let err = "";
let settled = false;
const settle = (fn: () => void) => {
if (settled) return;
settled = true;
clearTimeout(timeoutId);
if (signal && abortHandler) signal.removeEventListener("abort", abortHandler);
fn();
};
const kill = (message: string) => {
try { proc.kill("SIGKILL"); } catch {}
settle(() => reject(new Error(message)));
};
const timeoutId = setTimeout(() => kill("ffprobe timed out"), PROBE_TIMEOUT_MS);
const abortHandler = signal ? () => kill("ffprobe aborted") : null;
if (signal && abortHandler) {
if (signal.aborted) { kill("ffprobe aborted"); return; }
signal.addEventListener("abort", abortHandler, { once: true });
}
proc.stdout?.on("data", (d) => { out += d.toString(); });
proc.stderr?.on("data", (d) => { err += d.toString(); });
proc.on("error", (e) => settle(() => reject(e)));
proc.on("close", (code) => {
settle(() => {
if (code !== 0) {
reject(new Error(err.trim() || `ffprobe exited ${code}`));
return;
}
try {
const json = JSON.parse(out) as FfprobeJson;
const stream = json.streams?.[0] ?? {};
const streamBitrate = parseFiniteNumber(stream.bit_rate);
const formatBitrate = parseFiniteNumber(json.format?.bit_rate);
resolve({
durationSec: parseFiniteNumber(json.format?.duration),
videoCodec: typeof stream.codec_name === "string" ? stream.codec_name : null,
videoBFrames: parseNonNegativeNumber(stream.has_b_frames),
width: parseFiniteNumber(stream.width),
height: parseFiniteNumber(stream.height),
videoBitrate: streamBitrate ?? formatBitrate,
});
} catch (e) {
reject(e);
}
});
});
});
}
export async function probeVideoMetadata(file: VideoFile, signal?: AbortSignal): Promise<StoredVideoMetadata> {
const stat = await fs.stat(file.abs);
const existing = getStoredVideoMetadata(file.abs);
if (existing && isStatMatch(existing, stat.size, stat.mtimeMs)) {
if (existing.probeError || existing.probedAt != null) return existing;
}
const base = {
absPath: file.abs,
relPath: file.rel,
code: file.code,
sizeBytes: stat.size,
mtimeMs: stat.mtimeMs,
playbackMode: existing?.playbackMode ?? null,
};
try {
const probed = await runFfprobe(file.abs, signal);
rawDb.prepare(`
INSERT INTO video_metadata (
abs_path, rel_path, code, size_bytes, mtime_ms, probed_at, probe_error,
duration_sec, video_codec, video_b_frames, width, height, video_bitrate, playback_mode
) VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(abs_path) DO UPDATE SET
rel_path = excluded.rel_path,
code = excluded.code,
size_bytes = excluded.size_bytes,
mtime_ms = excluded.mtime_ms,
probed_at = excluded.probed_at,
probe_error = NULL,
duration_sec = excluded.duration_sec,
video_codec = excluded.video_codec,
video_b_frames = excluded.video_b_frames,
width = excluded.width,
height = excluded.height,
video_bitrate = excluded.video_bitrate,
playback_mode = excluded.playback_mode
`).run(
base.absPath, base.relPath, base.code, base.sizeBytes, base.mtimeMs, Date.now(),
probed.durationSec, probed.videoCodec, probed.videoBFrames, probed.width, probed.height, probed.videoBitrate, base.playbackMode,
);
} catch (e) {
rawDb.prepare(`
INSERT INTO video_metadata (
abs_path, rel_path, code, size_bytes, mtime_ms, probed_at, probe_error, playback_mode
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(abs_path) DO UPDATE SET
rel_path = excluded.rel_path,
code = excluded.code,
size_bytes = excluded.size_bytes,
mtime_ms = excluded.mtime_ms,
probed_at = excluded.probed_at,
probe_error = excluded.probe_error,
playback_mode = excluded.playback_mode
`).run(
base.absPath, base.relPath, base.code, base.sizeBytes, base.mtimeMs, Date.now(),
e instanceof Error ? e.message.slice(0, 500) : "ffprobe failed",
base.playbackMode,
);
}
return getStoredVideoMetadata(file.abs) ?? {
...base,
probedAt: null,
probeError: "metadata unavailable",
durationSec: null,
videoCodec: null,
videoBFrames: null,
width: null,
height: null,
videoBitrate: null,
partKind: null,
partIndex: null,
variantGroup: null,
};
}
export function setVideoPlaybackMode(file: VideoFile, mode: PlaybackMode | null): void {
rawDb.prepare(`
INSERT INTO video_metadata (abs_path, rel_path, code, size_bytes, mtime_ms, playback_mode)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(abs_path) DO UPDATE SET
rel_path = excluded.rel_path,
code = excluded.code,
size_bytes = excluded.size_bytes,
mtime_ms = excluded.mtime_ms,
playback_mode = excluded.playback_mode
`).run(file.abs, file.rel, file.code, file.size, file.mtime, mode);
}
export function formatDuration(sec: number | null | undefined): string | null {
if (sec == null || !Number.isFinite(sec) || sec <= 0) return null;
const total = Math.round(sec);
const h = Math.floor(total / 3600);
const m = Math.floor((total % 3600) / 60);
const s = total % 60;
if (h > 0) return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
return `${m}:${String(s).padStart(2, "0")}`;
}
export function formatBitrate(bps: number | null | undefined): string | null {
if (bps == null || !Number.isFinite(bps) || bps <= 0) return null;
if (bps >= 1_000_000) return `${(bps / 1_000_000).toFixed(1)} Mbps`;
if (bps >= 1_000) return `${Math.round(bps / 1_000)} Kbps`;
return `${Math.round(bps)} bps`;
}
export function formatBytes(bytes: number | null | undefined): string | null {
if (bytes == null || !Number.isFinite(bytes) || bytes < 0) return null;
const units = ["B", "KB", "MB", "GB", "TB"];
let n = bytes;
let i = 0;
while (n >= 1024 && i < units.length - 1) {
n /= 1024;
i++;
}
return `${i === 0 ? Math.round(n) : n.toFixed(n >= 10 ? 1 : 2)} ${units[i]}`;
}
export function formatResolution(width: number | null | undefined, height: number | null | undefined): string | null {
if (!width || !height) return null;
return `${width}x${height}`;
}
export function formatCodec(codec: string | null | undefined): string | null {
if (!codec) return null;
const map: Record<string, string> = {
h264: "H.264",
hevc: "HEVC",
h265: "HEVC",
av1: "AV1",
vp9: "VP9",
mpeg4: "MPEG-4",
};
return map[codec.toLowerCase()] ?? codec.toUpperCase();
}
export function formatVideoSummary(meta: StoredVideoMetadata | null | undefined): string | null {
if (!meta || meta.probeError) return null;
const parts = [
formatResolution(meta.width, meta.height),
formatCodec(meta.videoCodec),
formatBitrate(meta.videoBitrate),
formatBytes(meta.sizeBytes),
formatDuration(meta.durationSec),
].filter((part): part is string => Boolean(part));
return parts.length > 0 ? parts.join(" · ") : null;
}
+254
View File
@@ -0,0 +1,254 @@
/**
* Token-grammar classifier for video filenames in a JAVID group.
*
* Patterns use a simplified token grammar (option A1 from the mockups):
* - `{N}` — one or more digits, captured as the part index
* - `{L}` — single letter AZ, captured (A=1, B=2, ...)
* - everything else is a literal character
*
* Patterns match at the END of the filename stem (no extension),
* case-insensitive.
*
* Classification rules for files sharing one normalized JAV code:
* - "part" — stem ends with a configured pattern; index is the
* captured numeric/letter value.
* - "variant" — stem does NOT match any pattern but its prefix
* (first dot-segment) equals a stem that DID match.
* Variants belong to the matching part.
* - "single" — lone file in its code group with no pattern match.
*
* Tiebreak for "default variant" (the one to play first): the file
* whose stem equals the variant_group exactly. Otherwise the
* alphabetically first stem in the group.
*/
export interface CompiledPattern {
/** Original token-grammar source. */
source: string;
/** Compiled regex anchored to end-of-stem (case-insensitive). */
re: RegExp;
/** What the captured token represents. */
kind: "digits" | "letter";
}
/** Minimal description of one file presented to the classifier. */
export interface ClassifyInput {
/** Stable identifier, opaque to the classifier. */
key: string;
/** Filename stem (no extension), as on disk. */
stem: string;
}
export interface ClassifyResult {
key: string;
partKind: "part" | "variant" | "single";
/** 1-based sort index for parts; null otherwise. */
partIndex: number | null;
/** Stem-with-suffix-stripped — variants share this with their part. */
variantGroup: string | null;
}
const TOKEN_RE = /\{[NL]\}/g;
/** Compile one token-grammar pattern into a regex. Throws on bad token. */
export function compileToken(source: string): CompiledPattern | null {
if (!source) return null;
// Validate first: only {N} and {L} are allowed; nothing else may use {}.
// A bare `{` without a known token is invalid.
let kind: "digits" | "letter" | null = null;
let body = "";
let i = 0;
while (i < source.length) {
const c = source[i]!;
if (c === "{") {
const close = source.indexOf("}", i);
if (close < 0) return null;
const tok = source.slice(i, close + 1);
if (tok === "{N}") {
if (kind != null) return null; // only one capture per pattern
body += "(\\d+)";
kind = "digits";
} else if (tok === "{L}") {
if (kind != null) return null;
body += "([A-Za-z])";
kind = "letter";
} else {
return null;
}
i = close + 1;
} else {
body += c.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
i++;
}
}
if (kind == null) return null;
return {
source,
re: new RegExp(body + "$", "i"),
kind,
};
}
/** Compile a list of patterns; silently drops malformed ones. */
export function compilePatterns(sources: string[]): CompiledPattern[] {
const out: CompiledPattern[] = [];
for (const s of sources) {
const c = compileToken(s);
if (c) out.push(c);
}
return out;
}
function indexFromCapture(capture: string, kind: "digits" | "letter"): number | null {
if (kind === "digits") {
const n = Number(capture);
return Number.isFinite(n) && n > 0 ? Math.trunc(n) : null;
}
// Letter: A=1, B=2, ...
const code = capture.toUpperCase().charCodeAt(0);
if (code < 65 || code > 90) return null;
return code - 64;
}
interface PatternHit {
partIndex: number;
/** Stem with the matched suffix removed. */
variantGroup: string;
}
function tryMatch(stem: string, patterns: CompiledPattern[]): PatternHit | null {
for (const p of patterns) {
const m = stem.match(p.re);
if (!m) continue;
const idx = indexFromCapture(m[1] ?? "", p.kind);
if (idx == null) continue;
return {
partIndex: idx,
variantGroup: stem.slice(0, m.index!),
};
}
return null;
}
/**
* Classify a group of files that share one normalized JAV code.
*
* Algorithm:
* 1. Try each pattern against each stem; record matches.
* 2. Files with no match are candidate variants. A candidate is a
* variant of a matched file if its stem's first dot-segment
* equals the matched file's variant_group's first dot-segment.
* (This catches `XXX-001.fixed.mp4` aligning with `XXX-001-cd1.mp4`
* → no, those don't share a dot-prefix; they'd stay singles. But
* `XXX-001-cd1.fixed.mp4` would align with `XXX-001-cd1.mp4`.)
* 3. If no patterns match anything in the group, all stems share
* one variant_group (the longest common prefix of all stems,
* trimmed at the last alpha-numeric run); kind = variant for >1
* files, single for 1.
*/
export function classifyGroup(
files: ClassifyInput[],
patterns: CompiledPattern[],
): ClassifyResult[] {
if (files.length === 0) return [];
if (files.length === 1) {
const only = files[0]!;
return [{ key: only.key, partKind: "single", partIndex: null, variantGroup: null }];
}
// Pass 1: pattern match.
const hits = new Map<string, PatternHit>();
for (const f of files) {
const hit = tryMatch(f.stem, patterns);
if (hit) hits.set(f.key, hit);
}
if (hits.size === 0) {
// No part-style suffixes detected anywhere → treat the whole group
// as variants of one part.
const group = longestCommonPrefix(files.map((f) => f.stem));
return files.map((f) => ({
key: f.key,
partKind: "variant" as const,
partIndex: null,
variantGroup: group || f.stem,
}));
}
// Pass 2: attach unmatched stems to the matched stem they extend.
// A non-matching stem `S` is a variant of part group `G` iff `S`
// starts with `G + "."` (i.e. `G` followed by a dot — the typical
// "alt encode" suffix shape: `XXX-001-cd1.fixed.mp4`).
const matchedGroupKeys = Array.from(new Set(Array.from(hits.values()).map((h) => h.variantGroup)));
// Sort by length desc so longer (more specific) groups bind first.
matchedGroupKeys.sort((a, b) => b.length - a.length);
const out: ClassifyResult[] = [];
for (const f of files) {
const hit = hits.get(f.key);
if (hit) {
out.push({
key: f.key,
partKind: "part",
partIndex: hit.partIndex,
variantGroup: hit.variantGroup,
});
continue;
}
// Unmatched: try to attach to a part group via dot-prefix.
const attached = matchedGroupKeys.find(
(g) => g && (f.stem === g || f.stem.startsWith(g + ".")),
);
if (attached) {
out.push({ key: f.key, partKind: "variant", partIndex: null, variantGroup: attached });
} else {
// No way to attach — the file is a stray. Mark single.
out.push({ key: f.key, partKind: "single", partIndex: null, variantGroup: null });
}
}
return out;
}
function longestCommonPrefix(strs: string[]): string {
if (strs.length === 0) return "";
let prefix = strs[0]!;
for (let i = 1; i < strs.length; i++) {
const s = strs[i]!;
let j = 0;
while (j < prefix.length && j < s.length && prefix[j] === s[j]) j++;
prefix = prefix.slice(0, j);
if (!prefix) return "";
}
// Trim trailing punctuation so we don't end on a half-word like "XXX-001.".
return prefix.replace(/[\s._\-]+$/, "");
}
/**
* From a set of files all sharing the same variantGroup, pick the one
* to play by default. Rule: stem === group exactly; else alphabetically
* first.
*/
export function pickDefaultVariant<T extends { stem: string }>(
variants: T[],
group: string,
): T | null {
if (variants.length === 0) return null;
const exact = variants.find((v) => v.stem === group);
if (exact) return exact;
return [...variants].sort((a, b) => a.stem.localeCompare(b.stem))[0] ?? null;
}
/**
* Compute a short label for a variant relative to its group stem.
* `XXX-001.fixed` with group `XXX-001` → `fixed`.
* Falls back to `original` for the default / matching stem.
*/
export function variantLabel(stem: string, group: string): string {
if (stem === group) return "original";
if (stem.startsWith(group + ".")) {
return stem.slice(group.length + 1) || "original";
}
if (stem.startsWith(group)) {
return stem.slice(group.length).replace(/^[._\-\s]+/, "") || "original";
}
return stem;
}
+81
View File
@@ -0,0 +1,81 @@
import "server-only";
import path from "node:path";
import { getConfiguredVideoRoots } from "./index";
import { getAppSetting } from "@/lib/db/appSettings";
import { isManualSubtitlePath } from "./manualSubtitles";
/**
* In-process set of subtitle paths the user picked via /api/pick-file
* during this session. Covers the case where someone browses a .srt
* sitting outside any indexed video root — the OS picker IS the
* authorization. Entries time out after TTL_MS to bound how long an
* old picked path remains servable.
*/
const TTL_MS = 60 * 60 * 1000; // 1 hour
const trusted = new Map<string, number>();
function pruneExpired(now: number): void {
for (const [k, expiresAt] of trusted) {
if (expiresAt <= now) trusted.delete(k);
}
}
function normalize(p: string): string {
// Path keys use the resolved + lowercased form on Windows so case
// differences don't bypass the guard. POSIX is case-sensitive so we
// keep original case there.
const resolved = path.resolve(p);
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
}
export function trustSubtitlePath(abs: string): void {
pruneExpired(Date.now());
trusted.set(normalize(abs), Date.now() + TTL_MS);
}
export function isSessionTrustedSubtitlePath(abs: string): boolean {
const now = Date.now();
pruneExpired(now);
const key = normalize(abs);
const exp = trusted.get(key);
if (exp == null) return false;
if (exp <= now) {
trusted.delete(key);
return false;
}
return true;
}
function isInside(child: string, parent: string): boolean {
const c = process.platform === "win32" ? path.resolve(child).toLowerCase() : path.resolve(child);
const p = process.platform === "win32" ? path.resolve(parent).toLowerCase() : path.resolve(parent);
if (!p) return false;
if (c === p) return true;
const sep = path.sep;
return c.startsWith(p.endsWith(sep) ? p : p + sep);
}
/**
* True if `abs` resolves under one of:
* - a configured video root,
* - a configured subtitleExtraPaths entry,
* - the implicit data/generated-subtitles/ root (WhisperJAV output),
* - a session-trusted pick-file path (exact match, not prefix),
* - a path persisted in the manual_subtitles table (user explicitly
* Browse'd it during a previous session).
*/
export function isAllowedSubtitlePath(abs: string): boolean {
const resolved = path.resolve(abs);
for (const root of getConfiguredVideoRoots()) {
if (root && isInside(resolved, root)) return true;
}
const subRoots = getAppSetting("subtitleExtraPaths") ?? [];
for (const root of subRoots) {
if (root && isInside(resolved, root)) return true;
}
const generatedRoot = path.join(process.cwd(), "data", "generated-subtitles");
if (isInside(resolved, generatedRoot)) return true;
if (isSessionTrustedSubtitlePath(resolved)) return true;
if (isManualSubtitlePath(resolved)) return true;
return false;
}
+145
View File
@@ -0,0 +1,145 @@
import "server-only";
import path from "node:path";
import fs from "node:fs";
import fsp from "node:fs/promises";
import crypto from "node:crypto";
/**
* Bump on any change to srtToVtt, the ffmpeg arg recipe, or the
* cache-key composition. Old entries become unreachable automatically.
* v1 → initial.
* v2 → added decodeSubtitleBuffer for non-UTF-8 SRTs/VTTs (cp936,
* shift-jis, big5, UTF-16). Existing UTF-8-only entries would
* still be correct but the version bump ensures any cached
* output produced with a buggy decode path is regenerated.
*/
export const CONVERTER_VERSION = 2;
const CACHE_DIR = path.join(process.cwd(), "data", "subtitle-cache");
fs.mkdirSync(CACHE_DIR, { recursive: true });
export type CacheKind = "embedded" | "srt" | "ass" | "ssa";
export interface CacheKeyInput {
abs: string;
size: number;
mtimeMs: number;
kind: CacheKind;
/** ffmpeg stream index for embedded; ext for sidecar files. */
streamOrExt: string | number;
}
export function cachePath(input: CacheKeyInput): string {
const raw = [
input.abs,
input.size,
Math.round(input.mtimeMs),
input.kind,
String(input.streamOrExt),
CONVERTER_VERSION,
].join("|");
const hash = crypto.createHash("sha1").update(raw).digest("hex");
return path.join(CACHE_DIR, `${hash}.vtt`);
}
export async function readCache(file: string): Promise<Buffer | null> {
try {
const buf = await fsp.readFile(file);
// Bump mtime so LRU pruning treats this entry as recently used.
// Best effort: failure (read-only fs, locked file) is harmless.
const now = Date.now() / 1000;
fsp.utimes(file, now, now).catch(() => { /* ignore */ });
return buf;
} catch {
return null;
}
}
let writesSinceLastPrune = 0;
const PRUNE_WRITE_INTERVAL = 25;
export async function writeCache(file: string, data: Buffer | string): Promise<void> {
// Atomic via rename — avoids partial files if the process is killed
// mid-write or two requests race on the same key.
const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
try {
await fsp.writeFile(tmp, data);
await fsp.rename(tmp, file);
} catch (e) {
try { await fsp.unlink(tmp); } catch { /* ignore */ }
throw e;
}
writesSinceLastPrune++;
if (writesSinceLastPrune >= PRUNE_WRITE_INTERVAL) {
writesSinceLastPrune = 0;
void pruneSubtitleCacheIfNeeded();
}
}
interface PruneResult {
scanned: number;
removed: number;
beforeBytes: number;
afterBytes: number;
}
/** LRU sweep keyed on file mtime. Walks `data/subtitle-cache/`,
* computes total size, and if it exceeds the configured limit,
* deletes the oldest-mtime entries until size drops below 80% of
* the cap. No-op when the limit setting is 0 (unlimited). */
export async function pruneSubtitleCacheIfNeeded(): Promise<PruneResult> {
const { getAppSetting } = await import("@/lib/db/appSettings");
const limitMb = Number(getAppSetting("subtitleCacheLimitMb"));
const result: PruneResult = { scanned: 0, removed: 0, beforeBytes: 0, afterBytes: 0 };
if (!Number.isFinite(limitMb) || limitMb <= 0) return result;
const limitBytes = limitMb * 1024 * 1024;
const lowWatermark = Math.floor(limitBytes * 0.8);
let entries: import("node:fs").Dirent[];
try {
entries = await fsp.readdir(CACHE_DIR, { withFileTypes: true });
} catch {
return result;
}
type CacheEntry = { abs: string; size: number; mtimeMs: number };
const items: CacheEntry[] = [];
for (const e of entries) {
if (!e.isFile() || !e.name.endsWith(".vtt")) continue;
const abs = path.join(CACHE_DIR, e.name);
try {
const stat = await fsp.stat(abs);
items.push({ abs, size: stat.size, mtimeMs: stat.mtimeMs });
result.scanned++;
result.beforeBytes += stat.size;
} catch { /* file vanished mid-walk; skip */ }
}
if (result.beforeBytes <= limitBytes) {
result.afterBytes = result.beforeBytes;
return result;
}
// Oldest first.
items.sort((a, b) => a.mtimeMs - b.mtimeMs);
let running = result.beforeBytes;
for (const it of items) {
if (running <= lowWatermark) break;
try {
await fsp.unlink(it.abs);
running -= it.size;
result.removed++;
} catch { /* concurrent delete; skip */ }
}
result.afterBytes = running;
if (result.removed > 0) {
console.log(
`[subtitle-cache] pruned ${result.removed}/${result.scanned} files; ${(result.beforeBytes / 1_048_576).toFixed(1)}MB → ${(running / 1_048_576).toFixed(1)}MB`,
);
}
return result;
}
/** Bootstrap entrypoint — fire one sweep on module load (delayed so
* other startup work isn't blocked). */
function scheduleBootstrapPrune(): void {
setTimeout(() => { void pruneSubtitleCacheIfNeeded(); }, 5_000);
}
scheduleBootstrapPrune();
+195
View File
@@ -0,0 +1,195 @@
import "server-only";
import path from "node:path";
import fs from "node:fs/promises";
import iconv from "iconv-lite";
export const SUBTITLE_EXTS = [".srt", ".vtt", ".ass", ".ssa"] as const;
export type SubtitleExt = (typeof SUBTITLE_EXTS)[number];
const SUBTITLE_EXT_SET = new Set<string>(SUBTITLE_EXTS);
export type LangIso = "eng" | "zho" | "jpn";
export type LangPref = "EN" | "CN" | "JP" | "off";
export interface SubtitleFileEntry {
abs: string;
filename: string;
}
export async function walkSubtitles(root: string, maxDepth = 2): Promise<SubtitleFileEntry[]> {
const out: SubtitleFileEntry[] = [];
type Frame = { dir: string; depth: number };
const stack: Frame[] = [{ dir: root, depth: 0 }];
while (stack.length) {
const { dir, depth } = stack.pop()!;
let entries: import("node:fs").Dirent[];
try {
entries = await fs.readdir(dir, { withFileTypes: true });
} catch {
continue;
}
for (const e of entries) {
const full = path.join(dir, e.name);
if (e.isDirectory()) {
if (depth < maxDepth) stack.push({ dir: full, depth: depth + 1 });
} else if (e.isFile()) {
const ext = path.extname(e.name).toLowerCase();
if (SUBTITLE_EXT_SET.has(ext)) out.push({ abs: full, filename: e.name });
}
}
}
return out;
}
const PREF_TO_ISO: Record<Exclude<LangPref, "off">, LangIso> = {
EN: "eng",
CN: "zho",
JP: "jpn",
};
const ISO_TO_PREF: Record<LangIso, Exclude<LangPref, "off">> = {
eng: "EN",
zho: "CN",
jpn: "JP",
};
export function isoFromPref(pref: LangPref): LangIso | null {
return pref === "off" ? null : PREF_TO_ISO[pref];
}
export function prefFromIso(iso: LangIso | null): LangPref {
return iso == null ? "off" : ISO_TO_PREF[iso];
}
const ENGLISH_TOKENS = new Set(["en", "eng", "english"]);
const CHINESE_TOKENS = new Set([
"zh", "zho", "chi", "chs", "cht", "chn", "cn", "chinese",
"schinese", "tchinese", "simplified", "traditional",
"zh-cn", "zh-tw", "zh-hans", "zh-hant",
]);
const JAPANESE_TOKENS = new Set(["ja", "jp", "jpn", "japanese", "jap"]);
export function normalizeLanguageTag(tag: string | null | undefined): LangIso | null {
if (!tag) return null;
const lower = tag.trim().toLowerCase();
if (!lower) return null;
if (ENGLISH_TOKENS.has(lower)) return "eng";
if (CHINESE_TOKENS.has(lower)) return "zho";
if (JAPANESE_TOKENS.has(lower)) return "jpn";
return null;
}
export function languageDisplay(iso: LangIso | null): string {
if (iso === "eng") return "English";
if (iso === "zho") return "Chinese";
if (iso === "jpn") return "Japanese";
return "Unknown";
}
const TOKEN_SPLIT = /[\s._\-\[\]()+,;]+/g;
export interface DetectedLanguage {
/** Single ISO code if exactly one language was detected. */
lang: LangIso | null;
/** Display label — "English", "Chinese", "English/Chinese", "Unknown". */
label: string;
}
/** Inspect a filename's stem for embedded language hints. Multiple hits
* produce a compound label (e.g. "English/Chinese") but `lang` stays null
* so sticky-pref matching only ever resolves to a single language. */
export function detectLanguageFromName(filename: string): DetectedLanguage {
const ext = path.extname(filename).toLowerCase();
const stem = ext ? filename.slice(0, -ext.length) : filename;
const tokens = stem.toLowerCase().split(TOKEN_SPLIT).filter(Boolean);
const found = new Set<LangIso>();
for (const t of tokens) {
const iso = normalizeLanguageTag(t);
if (iso) found.add(iso);
}
if (found.size === 0) return { lang: null, label: "Unknown" };
if (found.size === 1) {
const iso = [...found][0]!;
return { lang: iso, label: languageDisplay(iso) };
}
const order: LangIso[] = ["eng", "zho", "jpn"];
const ordered = order.filter((i) => found.has(i));
return { lang: null, label: ordered.map(languageDisplay).join("/") };
}
const SRT_TIMESTAMP = /(\d{1,2}:\d{2}:\d{2}),(\d{3})/g;
/** Pure JS SRT → WebVTT converter. Strips BOM, normalizes CRLF, swaps
* the comma in HH:MM:SS,mmm timestamps for a dot, and prepends the
* WEBVTT header. No styling translation. Cheap; runs on every sidecar
* miss without spawning ffmpeg. */
export function srtToVtt(srt: string): string {
let body = srt;
if (body.charCodeAt(0) === 0xfeff) body = body.slice(1);
body = body.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
body = body.replace(SRT_TIMESTAMP, "$1.$2");
return `WEBVTT\n\n${body.trimStart()}`;
}
export function stemOf(filename: string): string {
const ext = path.extname(filename);
return ext ? filename.slice(0, -ext.length) : filename;
}
const REPLACEMENT_CHAR = "";
/**
* Decode a subtitle file buffer to a JS string with best-effort
* encoding detection. Many older Asian SRTs ship as cp936/GBK or
* Shift-JIS — feeding them through `Buffer.toString("utf8")` produces
* mojibake. Strategy:
* 1. Strip BOM if present (UTF-8 / UTF-16 LE / UTF-16 BE).
* 2. Try UTF-8 strict. If it decodes without invalid sequences, use it.
* 3. Otherwise decode as UTF-8 / shift_jis / gb18030 / big5 and
* pick whichever has the fewest replacement chars per kbyte.
* 4. Tie-break preference: shift_jis when katakana/hiragana ranges
* appear in the JS surrogates, gb18030 otherwise — common
* heuristic for JP vs CN fansub source material.
*/
export function decodeSubtitleBuffer(buf: Buffer): string {
// BOM detection — if present, the encoding is unambiguous.
if (buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf) {
return buf.subarray(3).toString("utf8");
}
if (buf.length >= 2 && buf[0] === 0xff && buf[1] === 0xfe) {
return iconv.decode(buf.subarray(2), "utf-16le");
}
if (buf.length >= 2 && buf[0] === 0xfe && buf[1] === 0xff) {
return iconv.decode(buf.subarray(2), "utf-16be");
}
// UTF-8 strict — fast path for the common case.
try {
const decoder = new TextDecoder("utf-8", { fatal: true });
return decoder.decode(buf);
} catch { /* fall through to heuristic */ }
// Compare candidate encodings by replacement-char count.
const candidates: Array<"utf8" | "shift_jis" | "gb18030" | "big5"> = [
"utf8", "shift_jis", "gb18030", "big5",
];
let best: { encoding: typeof candidates[number]; text: string; score: number } | null = null;
for (const encoding of candidates) {
const text = iconv.decode(buf, encoding);
let bad = 0;
for (let i = 0; i < text.length; i++) {
if (text[i] === REPLACEMENT_CHAR) bad++;
}
// Tie-break preference: shift_jis when text contains kana, since
// gb18030 happens to map many JP code points without errors but
// produces gibberish that we wouldn't catch by rep-count alone.
const hasKana = /[぀-ヿ]/.test(text);
const adjusted = hasKana && encoding === "shift_jis"
? bad - 1
: encoding === "utf8" ? bad - 1 : bad;
if (best == null || adjusted < best.score) {
best = { encoding, text, score: adjusted };
}
}
return best?.text ?? buf.toString("utf8");
}
+202
View File
@@ -0,0 +1,202 @@
import "server-only";
import { rawDb } from "@/lib/db/client";
import type { JobRow, JobStatus } from "./types";
interface JobDbRow {
id: string;
code: string;
video_abs: string;
job_dir: string;
target_subtitle_path: string | null;
status: JobStatus;
enqueued_at: number;
started_at: number | null;
ended_at: number | null;
exit_code: number | null;
error: string | null;
stage: string | null;
stage_index: number | null;
stage_total: number | null;
cue_count: number | null;
cli_args: string;
log_path: string;
stats_path: string | null;
video_duration_sec: number | null;
mode: string | null;
}
function rowFromDb(r: JobDbRow): JobRow {
return {
id: r.id,
code: r.code,
videoAbs: r.video_abs,
jobDir: r.job_dir,
targetSubtitlePath: r.target_subtitle_path,
status: r.status,
enqueuedAt: r.enqueued_at,
startedAt: r.started_at,
endedAt: r.ended_at,
exitCode: r.exit_code,
error: r.error,
stage: r.stage,
stageIndex: r.stage_index,
stageTotal: r.stage_total,
cueCount: r.cue_count,
cliArgs: r.cli_args,
logPath: r.log_path,
statsPath: r.stats_path,
videoDurationSec: r.video_duration_sec,
mode: r.mode,
};
}
export function insertJob(row: Omit<JobRow, "startedAt" | "endedAt" | "exitCode" | "error" | "stage" | "stageIndex" | "stageTotal" | "cueCount" | "targetSubtitlePath">): void {
rawDb.prepare(`
INSERT INTO whisperjav_jobs (
id, code, video_abs, job_dir, target_subtitle_path, status,
enqueued_at, started_at, ended_at, exit_code, error,
stage, stage_index, stage_total, cue_count,
cli_args, log_path, stats_path, video_duration_sec, mode
) VALUES (?, ?, ?, ?, NULL, ?, ?, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ?, ?, ?, ?, ?)
`).run(
row.id, row.code, row.videoAbs, row.jobDir, row.status, row.enqueuedAt,
row.cliArgs, row.logPath, row.statsPath, row.videoDurationSec, row.mode,
);
}
/** Returns avg(elapsed_sec / video_duration_sec) over recent
* successful jobs for the given mode. Used to estimate remaining time
* for an in-flight job. Falls back to a per-mode seed when no history
* is available. */
export function estimateRealtimeMultiplier(mode: string): number {
const rows = rawDb
.prepare(
`SELECT started_at, ended_at, video_duration_sec
FROM whisperjav_jobs
WHERE status IN ('completed', 'warning')
AND mode = ?
AND started_at IS NOT NULL
AND ended_at IS NOT NULL
AND video_duration_sec IS NOT NULL
AND video_duration_sec > 0
ORDER BY ended_at DESC
LIMIT 10`,
)
.all(mode) as Array<{ started_at: number; ended_at: number; video_duration_sec: number }>;
if (rows.length === 0) {
if (mode === "fast") return 0.8;
if (mode === "qwen") return 6.0;
return 2.0; // balanced default
}
let sum = 0;
let n = 0;
for (const r of rows) {
const elapsed = (r.ended_at - r.started_at) / 1000;
if (elapsed <= 0) continue;
sum += elapsed / r.video_duration_sec;
n++;
}
return n > 0 ? sum / n : 2.0;
}
export function getJob(id: string): JobRow | null {
const r = rawDb.prepare(`SELECT * FROM whisperjav_jobs WHERE id = ?`).get(id) as JobDbRow | undefined;
return r ? rowFromDb(r) : null;
}
export function listJobsForCode(code: string, limit = 5): JobRow[] {
const rows = rawDb.prepare(`
SELECT * FROM whisperjav_jobs WHERE code = ?
ORDER BY enqueued_at DESC LIMIT ?
`).all(code, limit) as JobDbRow[];
return rows.map(rowFromDb);
}
/** Earliest queued job, regardless of code. */
export function nextQueuedJob(): JobRow | null {
const r = rawDb.prepare(`
SELECT * FROM whisperjav_jobs WHERE status = 'queued'
ORDER BY enqueued_at ASC LIMIT 1
`).get() as JobDbRow | undefined;
return r ? rowFromDb(r) : null;
}
/** Most recent non-terminal (queued/running) job for a code, if any. */
export function activeJobForCode(code: string): JobRow | null {
const r = rawDb.prepare(`
SELECT * FROM whisperjav_jobs
WHERE code = ? AND status IN ('queued','running')
ORDER BY enqueued_at DESC LIMIT 1
`).get(code) as JobDbRow | undefined;
return r ? rowFromDb(r) : null;
}
export function setStatus(id: string, status: JobStatus, fields: Partial<{
startedAt: number | null;
endedAt: number | null;
exitCode: number | null;
error: string | null;
targetSubtitlePath: string | null;
cueCount: number | null;
}> = {}): void {
const sets: string[] = ["status = ?"];
const args: (string | number | null)[] = [status];
const map: Record<string, string> = {
startedAt: "started_at",
endedAt: "ended_at",
exitCode: "exit_code",
error: "error",
targetSubtitlePath: "target_subtitle_path",
cueCount: "cue_count",
};
for (const [k, col] of Object.entries(map)) {
if (k in fields) {
sets.push(`${col} = ?`);
args.push((fields as Record<string, string | number | null>)[k] ?? null);
}
}
args.push(id);
rawDb.prepare(`UPDATE whisperjav_jobs SET ${sets.join(", ")} WHERE id = ?`).run(...args);
}
export function updateProgress(id: string, stage: string | null, idx: number | null, total: number | null): void {
rawDb.prepare(`
UPDATE whisperjav_jobs SET stage = ?, stage_index = ?, stage_total = ? WHERE id = ?
`).run(stage, idx, total, id);
}
/** Rows older than `cutoffMs` whose status is one of the terminal
* retention candidates (failed/cancelled). Used by the retention
* sweep to find job dirs to delete. */
export function listAgedTerminalJobs(cutoffMs: number): Array<{ id: string; jobDir: string }> {
const rows = rawDb.prepare(`
SELECT id, job_dir FROM whisperjav_jobs
WHERE status IN ('failed', 'cancelled')
AND COALESCE(ended_at, enqueued_at) < ?
`).all(cutoffMs) as Array<{ id: string; job_dir: string }>;
return rows.map((r) => ({ id: r.id, jobDir: r.job_dir }));
}
/** Used by the "Clear all job history" Settings action. */
export function listAllJobDirs(): string[] {
const rows = rawDb.prepare(`SELECT job_dir FROM whisperjav_jobs`).all() as Array<{ job_dir: string }>;
return rows.map((r) => r.job_dir);
}
export function deleteAllJobs(): number {
const result = rawDb.prepare(`DELETE FROM whisperjav_jobs WHERE status NOT IN ('queued', 'running')`).run();
return result.changes ?? 0;
}
/** Mark any running rows as failed (their child processes are dead).
* Queued rows remain queued — they're still waiting their turn. */
export function recoverOrphanedJobs(): number {
const result = rawDb.prepare(`
UPDATE whisperjav_jobs
SET status = 'failed',
error = 'process did not survive restart',
ended_at = ?
WHERE status = 'running'
`).run(Date.now());
return result.changes ?? 0;
}
+438
View File
@@ -0,0 +1,438 @@
import "server-only";
import path from "node:path";
import fs from "node:fs";
import fsp from "node:fs/promises";
import readline from "node:readline";
import { Readable } from "node:stream";
import { getAppSetting } from "@/lib/db/appSettings";
import { rawDb } from "@/lib/db/client";
import {
insertJob,
getJob,
setStatus,
updateProgress,
nextQueuedJob,
recoverOrphanedJobs,
activeJobForCode,
listAgedTerminalJobs,
listAllJobDirs,
deleteAllJobs,
} from "./db";
import {
buildJobArgs,
parseStageLine,
spawnJob,
validateOutcome,
moveFile,
isDirWritable,
jobBaseDir,
jobDirFor,
newJobId,
} from "./spawn";
import { findVideosForCode, rescanVideoIndex } from "@/lib/video";
import { getStoredVideoMetadata } from "@/lib/video/metadata";
import type { JobRow } from "./types";
declare global {
// eslint-disable-next-line no-var
var __whisperjavWorkerStarted: boolean | undefined;
// eslint-disable-next-line no-var
var __whisperjavRunningKill: ((reason: "cancel") => void) | undefined;
// eslint-disable-next-line no-var
var __whisperjavRunningId: string | null | undefined;
}
if (!global.__whisperjavWorkerStarted) {
global.__whisperjavWorkerStarted = true;
global.__whisperjavRunningKill = undefined;
global.__whisperjavRunningId = null;
// Restart sweep: any rows in `running` from a prior process are dead.
try {
const recovered = recoverOrphanedJobs();
if (recovered > 0) {
// eslint-disable-next-line no-console
console.log(`[whisperjav] recovered ${recovered} orphaned job(s) on bootstrap`);
}
} catch (e) {
// eslint-disable-next-line no-console
console.error("[whisperjav] orphan recovery failed:", e);
}
}
export interface EnqueueResult {
jobId: string;
}
export interface EnqueueAlreadyExists {
alreadyExists: true;
abs: string;
}
const BANNER_CHARS = /^[╔╗╚╝═║│┌┐└┘─\s]*$/;
const NOISE = [
/RequestsDependencyWarning/i,
/urllib3 \(\d+\.\d+/,
/chardet|charset_normalizer/,
/You are about to download and run code from an untrusted repository/,
/^Downloading: /,
/UserWarning:/,
/^\s*warnings\.warn/,
/^_check_repo_is_trusted/,
];
function isNoiseLine(s: string): boolean {
if (!s.trim()) return true;
if (BANNER_CHARS.test(s)) return true;
for (const re of NOISE) if (re.test(s)) return true;
return false;
}
function generatedSubtitlesDir(code: string): string {
return path.join(process.cwd(), "data", "generated-subtitles", code);
}
/** Compute the destination dir for a given video, given settings.
* Returns the dir absolute path. Creates it if needed. */
async function resolveDestDir(videoAbs: string, code: string): Promise<string> {
const s = getAppSetting("whisperjav");
if (s.outputLocation === "beside-video") {
const dir = path.dirname(videoAbs);
if (await isDirWritable(dir)) return dir;
}
const fallback = generatedSubtitlesDir(code);
await fsp.mkdir(fallback, { recursive: true });
return fallback;
}
/** Filename pattern WhisperJAV emits — derived from settings, used for
* pre-flight idempotency checks. The CLI may add a suffix beyond this
* pattern but the language tag is stable. */
function expectedSubtitleStemPrefix(videoStem: string, langTag: string): string {
return `${videoStem}.${langTag}`;
}
function langTagForSettings(): "ja" | "ko" | "zh" | "en" {
const s = getAppSetting("whisperjav");
if (s.outputMode === "direct-to-english") return "en";
switch (s.sourceLanguage) {
case "japanese": return "ja";
case "korean": return "ko";
case "chinese": return "zh";
case "english": return "en";
}
}
async function existingGeneratedFor(videoAbs: string, code: string): Promise<string | null> {
const langTag = langTagForSettings();
const stem = path.basename(videoAbs, path.extname(videoAbs));
const prefix = expectedSubtitleStemPrefix(stem, langTag);
const candidates = [
path.dirname(videoAbs),
generatedSubtitlesDir(code),
];
for (const dir of candidates) {
let entries: import("node:fs").Dirent[];
try {
entries = await fsp.readdir(dir, { withFileTypes: true });
} catch {
continue;
}
for (const e of entries) {
if (!e.isFile()) continue;
const lower = e.name.toLowerCase();
if (!lower.endsWith(".srt")) continue;
// Match `<stem>.<lang>` followed by ANY further token before .srt.
// Catches both `<stem>.<lang>.srt` and `<stem>.<lang>.whisperjav.srt`.
if (lower.startsWith(prefix.toLowerCase() + ".") || lower === `${prefix.toLowerCase()}.srt`) {
return path.join(dir, e.name);
}
}
}
return null;
}
export async function enqueueJob(opts: { code: string; partIdx: number; overwrite?: boolean }): Promise<EnqueueResult | EnqueueAlreadyExists> {
const settings = getAppSetting("whisperjav");
if (!settings.cliPath) {
throw new Error("WhisperJAV CLI path not configured");
}
let files = findVideosForCode(opts.code);
if (files.length === 0) {
// Index might be empty in dev — kick a rescan once.
await rescanVideoIndex();
files = findVideosForCode(opts.code);
}
const variant = files[opts.partIdx];
if (!variant) throw new Error(`No video found for code=${opts.code} part=${opts.partIdx}`);
const existing = await existingGeneratedFor(variant.abs, opts.code);
if (existing) {
if (!opts.overwrite) {
return { alreadyExists: true, abs: existing };
}
// User confirmed overwrite — remove the prior generated file so the
// post-run move at the queue worker doesn't trip its collision guard.
try {
await fsp.unlink(existing);
} catch (e) {
throw new Error(`Could not remove existing subtitle: ${(e as Error).message}`);
}
}
const id = newJobId();
const jobDir = jobDirFor(id);
await fsp.mkdir(jobDir, { recursive: true });
const statsPath = path.join(jobDir, "stats.json");
const logPath = path.join(jobDir, "stderr.log");
const args = buildJobArgs({
videoAbs: variant.abs,
outputDir: jobDir,
statsPath,
settings,
});
// Capture duration + mode at enqueue so the running-job UI can show
// an ETA. Duration may be missing if the file hasn't been probed yet
// — that's fine, ETA is best-effort.
const stored = getStoredVideoMetadata(variant.abs);
const videoDurationSec = stored?.durationSec && stored.durationSec > 0 ? stored.durationSec : null;
insertJob({
id,
code: opts.code,
videoAbs: variant.abs,
jobDir,
status: "queued",
enqueuedAt: Date.now(),
cliArgs: JSON.stringify(args),
logPath,
statsPath,
videoDurationSec,
mode: settings.quality,
});
scheduleTick();
return { jobId: id };
}
export function cancelJob(id: string): boolean {
const job = getJob(id);
if (!job) return false;
if (job.status === "queued") {
// Atomic flip: if the worker just picked this job up between our
// read and write, the WHERE clause matches zero rows and we fall
// through to the running branch.
const info = rawDb.prepare(
`UPDATE whisperjav_jobs SET status = 'cancelled', ended_at = ? WHERE id = ? AND status = 'queued'`,
).run(Date.now(), id);
if (info.changes > 0) return true;
}
// Re-read after the failed conditional update — status may now be 'running'.
const fresh = getJob(id) ?? job;
if (fresh.status === "running" && global.__whisperjavRunningId === id && global.__whisperjavRunningKill) {
global.__whisperjavRunningKill("cancel");
return true;
}
return false;
}
let tickPending = false;
function scheduleTick(): void {
if (tickPending) return;
tickPending = true;
// Defer to next tick so callers' DB writes are visible.
setImmediate(() => {
tickPending = false;
void runOne();
});
}
let workerBusy = false;
async function runOne(): Promise<void> {
if (workerBusy) return;
workerBusy = true;
try {
while (true) {
const next = nextQueuedJob();
if (!next) break;
await processJob(next);
}
} finally {
workerBusy = false;
}
}
async function processJob(job: JobRow): Promise<void> {
const settings = getAppSetting("whisperjav");
if (!settings.cliPath) {
setStatus(job.id, "failed", { startedAt: Date.now(), endedAt: Date.now(), error: "WhisperJAV CLI path cleared while job was queued" });
return;
}
setStatus(job.id, "running", { startedAt: Date.now() });
let args: string[];
try {
args = JSON.parse(job.cliArgs) as string[];
} catch {
setStatus(job.id, "failed", { endedAt: Date.now(), error: "stored cli_args malformed" });
return;
}
const logStream = fs.createWriteStream(job.logPath, { flags: "a" });
const spawned = spawnJob(settings.cliPath, args);
global.__whisperjavRunningId = job.id;
let cancelled = false;
global.__whisperjavRunningKill = (reason) => {
if (reason === "cancel") cancelled = true;
spawned.kill();
};
// Stage parser — write-through on every match. WhisperJAV emits
// at most a handful of "Step N/M:" lines per job; the prior
// debounce raced the rl close drain and dropped the final stage.
const rl = readline.createInterface({ input: spawned.proc.stderr ?? Readable.from([]) });
rl.on("line", (raw: string) => {
// Strip CSI escape sequences (optional ESC byte + "[...m").
const line = raw.replace(/?\[[0-9;]*m/g, "");
logStream.write(line + "\n");
if (isNoiseLine(line)) return;
const stage = parseStageLine(line);
if (stage) {
updateProgress(job.id, stage.stage, stage.index, stage.total);
}
});
// We don't currently expect anything on stdout; tee anyway.
spawned.proc.stdout?.on("data", (b) => { logStream.write(b); });
const exitCode: number | null = await new Promise((resolve) => {
spawned.proc.on("close", (code) => resolve(code));
spawned.proc.on("error", () => resolve(null));
});
rl.close();
await new Promise<void>((res) => logStream.end(() => res()));
global.__whisperjavRunningKill = undefined;
global.__whisperjavRunningId = null;
if (cancelled) {
setStatus(job.id, "cancelled", { endedAt: Date.now(), exitCode });
// Keep job dir for diagnosis. Could prune later via retention sweep.
return;
}
const result = await validateOutcome({
exitCode,
statsPath: job.statsPath ?? path.join(job.jobDir, "stats.json"),
jobDir: job.jobDir,
});
if (!result.success) {
setStatus(job.id, "failed", { endedAt: Date.now(), exitCode, error: result.reason ?? "validation failed" });
return;
}
// Move the .srt to its final destination, fail loudly on collision.
if (!result.finalSrtPath) {
setStatus(job.id, "failed", { endedAt: Date.now(), exitCode, error: "no final_srt path resolved" });
return;
}
let destDir: string;
try {
destDir = await resolveDestDir(job.videoAbs, job.code);
} catch (e) {
setStatus(job.id, "failed", { endedAt: Date.now(), exitCode, error: `dest dir resolve failed: ${(e as Error).message}` });
return;
}
const destFile = path.join(destDir, path.basename(result.finalSrtPath));
if (fs.existsSync(destFile)) {
setStatus(job.id, "failed", { endedAt: Date.now(), exitCode, error: `Output already exists at ${destFile}` });
return;
}
try {
await moveFile(result.finalSrtPath, destFile);
} catch (e) {
setStatus(job.id, "failed", { endedAt: Date.now(), exitCode, error: `move failed: ${(e as Error).message}` });
return;
}
// Cleanup temp dir on success/warning.
try {
await fsp.rm(job.jobDir, { recursive: true, force: true });
} catch { /* best effort */ }
setStatus(job.id, result.warning ? "warning" : "completed", {
endedAt: Date.now(),
exitCode,
targetSubtitlePath: destFile,
cueCount: result.cueCount,
error: result.warning ? result.reason : null,
});
// Opportunistic retention sweep — keeps the disk tidy without a
// separate scheduler. No-op when retention is 0.
void runRetentionSweep();
}
/** Mark every queued (not-yet-running) job cancelled. Used by the
* "Stop Batch" button to drain the queue without touching the
* currently-running job. Returns the count cancelled. */
export function cancelAllQueued(): number {
const result = rawDb.prepare(`
UPDATE whisperjav_jobs SET status = 'cancelled', ended_at = ?
WHERE status = 'queued'
`).run(Date.now());
return result.changes ?? 0;
}
/** Module-load side effect: make sure leftover queued jobs from a
* prior process get picked up. Safe to call repeatedly. */
export function bootstrapQueue(): void {
scheduleTick();
void runRetentionSweep();
}
/** Delete failed/cancelled job dirs older than `retentionDays`. Always
* safe to call — no-ops when retention is 0 or no aged rows exist. */
export async function runRetentionSweep(): Promise<{ removed: number }> {
const settings = getAppSetting("whisperjav");
const days = Number(settings.retentionDays);
if (!Number.isFinite(days) || days <= 0) return { removed: 0 };
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
const aged = listAgedTerminalJobs(cutoff);
let removed = 0;
for (const row of aged) {
try {
await fsp.rm(row.jobDir, { recursive: true, force: true });
removed++;
} catch (e) {
console.error(`[whisperjav] failed to prune ${row.jobDir}:`, e);
}
}
if (removed > 0) {
console.log(`[whisperjav] retention sweep removed ${removed} job dir(s)`);
}
return { removed };
}
/** Wipe every non-running job row + every temp dir on disk. Used by
* the "Clear all job history" Settings action. Returns counts. */
export async function clearAllJobHistory(): Promise<{ rows: number; dirs: number }> {
const dirs = listAllJobDirs();
let dirsRemoved = 0;
for (const dir of dirs) {
try {
await fsp.rm(dir, { recursive: true, force: true });
dirsRemoved++;
} catch { /* best effort */ }
}
const rows = deleteAllJobs();
return { rows, dirs: dirsRemoved };
}
bootstrapQueue();
export { activeJobForCode, getJob, jobBaseDir };
+314
View File
@@ -0,0 +1,314 @@
import "server-only";
import { spawn, type ChildProcess } from "node:child_process";
import path from "node:path";
import fs from "node:fs";
import fsp from "node:fs/promises";
import { randomUUID } from "node:crypto";
import type { WhisperJavSettings } from "@/lib/db/appSettings";
const VERIFY_TIMEOUT_MS = 30_000;
const BANNER_CHARS = /^[╔╗╚╝═║│┌┐└┘─\s]*$/;
const NOISE_PATTERNS: RegExp[] = [
/RequestsDependencyWarning/i,
/urllib3 \(\d+\.\d+/,
/chardet|charset_normalizer/,
/You are about to download and run code from an untrusted repository/,
/^Downloading: /,
/UserWarning:/,
/^\s*warnings\.warn/,
/^_check_repo_is_trusted/,
/^\s*$/, // blank
];
function isNoise(line: string): boolean {
if (BANNER_CHARS.test(line)) return true;
for (const re of NOISE_PATTERNS) {
if (re.test(line)) return true;
}
return false;
}
const STAGE_RE = /Step\s+(\d+)\s*\/\s*(\d+):\s*(.+?)\s*$/;
export interface ParsedStage {
index: number;
total: number;
stage: string;
}
export function parseStageLine(line: string): ParsedStage | null {
const m = line.match(STAGE_RE);
if (!m) return null;
const index = Number(m[1]);
const total = Number(m[2]);
if (!Number.isFinite(index) || !Number.isFinite(total)) return null;
return { index, total, stage: m[3]!.trim() };
}
const VERSION_RE = /WhisperJAV\s+(\d+\.\d+\.\d+)/;
export interface VerifyResult {
ok: boolean;
version?: string;
resolvedPath?: string;
error?: string;
}
/** Run `<cliPath> --version` and parse stdout. Stderr ignored
* (RequestsDependencyWarning is benign). */
export function verifyCli(cliPath: string): Promise<VerifyResult> {
return new Promise((resolve) => {
const t0 = Date.now();
let proc: ChildProcess;
try {
proc = spawn(cliPath, ["--version"], { stdio: ["ignore", "pipe", "pipe"] });
} catch (e) {
console.error(`[whisperjav verify] spawn failed (${Date.now() - t0}ms):`, (e as Error).message);
resolve({ ok: false, error: (e as Error).message });
return;
}
let stdout = "";
let stderr = "";
let settled = false;
const settle = (val: VerifyResult) => {
if (settled) return;
settled = true;
clearTimeout(t);
console.log(`[whisperjav verify] ${val.ok ? "ok" : "fail"} in ${Date.now() - t0}ms`);
resolve(val);
};
const t = setTimeout(() => {
try { proc.kill("SIGKILL"); } catch {}
console.error(`[whisperjav verify] timeout after ${VERIFY_TIMEOUT_MS}ms; stdout="${stdout.trim()}" stderr_tail="${stderr.trim().split("\n").slice(-3).join(" | ")}"`);
settle({ ok: false, error: "verify timed out" });
}, VERIFY_TIMEOUT_MS);
proc.stdout?.on("data", (d) => { stdout += d.toString(); });
proc.stderr?.on("data", (d) => { stderr += d.toString(); });
proc.on("error", (e) => settle({ ok: false, error: e.message }));
proc.on("close", () => {
const merged = stdout + "\n" + stderr;
const m = merged.match(VERSION_RE);
if (m) {
settle({ ok: true, version: m[1], resolvedPath: cliPath });
return;
}
settle({ ok: false, error: stderr.trim().split("\n").slice(-3).join("\n") || "no version detected in output" });
});
});
}
/** Look up the CLI on PATH via the OS-specific where/which. Returns
* the first match or null. */
export async function autoDetectCli(): Promise<string | null> {
return new Promise((resolve) => {
const cmd = process.platform === "win32" ? "where" : "which";
let proc: ChildProcess;
try {
proc = spawn(cmd, ["whisperjav"], { stdio: ["ignore", "pipe", "ignore"] });
} catch {
resolve(null);
return;
}
let stdout = "";
proc.stdout?.on("data", (d) => { stdout += d.toString(); });
proc.on("error", () => resolve(null));
proc.on("close", () => {
const first = stdout.split(/\r?\n/).map((l) => l.trim()).find(Boolean);
resolve(first ?? null);
});
});
}
const QUALITY_TO_MODE: Record<WhisperJavSettings["quality"], string> = {
fast: "fast",
balanced: "balanced",
qwen: "qwen",
};
/** Build the argv (without the program name) for a generation job. */
export function buildJobArgs(opts: {
videoAbs: string;
outputDir: string;
statsPath: string;
settings: WhisperJavSettings;
}): string[] {
const args: string[] = [
opts.videoAbs,
"--mode", QUALITY_TO_MODE[opts.settings.quality],
"--language", opts.settings.sourceLanguage,
"--subs-language", opts.settings.outputMode,
"--sensitivity", opts.settings.sensitivity,
"--output-dir", opts.outputDir,
"--no-progress",
"--verbosity", "summary",
"--stats-file", opts.statsPath,
];
if (opts.settings.noSignature) args.push("--no-signature");
return args;
}
export interface SpawnedJob {
proc: ChildProcess;
/** Best-effort kill that takes Python child workers down too. */
kill: () => void;
}
/** Spawn a generation job. Caller wires stderr/stdout consumers. */
export function spawnJob(cliPath: string, args: string[]): SpawnedJob {
const proc = spawn(cliPath, args, { stdio: ["ignore", "pipe", "pipe"] });
return {
proc,
kill: () => killTree(proc),
};
}
function killTree(proc: ChildProcess): void {
if (!proc.pid) return;
if (process.platform === "win32") {
// taskkill /T cascades to children. /F forces. Spawn fire-and-forget.
try {
spawn("taskkill", ["/pid", String(proc.pid), "/T", "/F"], { stdio: "ignore" });
} catch { /* ignore */ }
return;
}
try { proc.kill("SIGTERM"); } catch { /* ignore */ }
// Escalate after a short grace period.
setTimeout(() => {
try { proc.kill("SIGKILL"); } catch { /* ignore */ }
}, 3000);
}
export interface StatsEntry {
status?: string;
metadata?: {
output_files?: { final_srt?: string };
summary?: { final_subtitles_refined?: number };
errors?: unknown[];
};
}
export interface ValidationResult {
/** Strict success — exit 0, stats success, errors empty, srt exists. */
success: boolean;
/** Set when success === true but cue count is 0 (warning state). */
warning: boolean;
finalSrtPath: string | null;
cueCount: number | null;
/** Human-readable reason when validation failed. */
reason: string | null;
}
/** Apply the success criteria from the plan. Caller passes exit code
* and the stats path; we read + parse + check. */
export async function validateOutcome(opts: {
exitCode: number | null;
statsPath: string;
jobDir: string;
}): Promise<ValidationResult> {
if (opts.exitCode !== 0) {
return { success: false, warning: false, finalSrtPath: null, cueCount: null, reason: `exit code ${opts.exitCode}` };
}
let raw: string;
try {
raw = await fsp.readFile(opts.statsPath, "utf8");
} catch {
// Stats missing fallback: accept if exactly one .srt exists in jobDir.
const stray = await findSingleSrt(opts.jobDir);
if (stray) {
return {
success: true,
warning: true,
finalSrtPath: stray,
cueCount: null,
reason: "stats unavailable, accepted by file presence",
};
}
return { success: false, warning: false, finalSrtPath: null, cueCount: null, reason: "stats.json missing and no .srt found" };
}
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
return { success: false, warning: false, finalSrtPath: null, cueCount: null, reason: "stats.json malformed" };
}
const arr = Array.isArray(parsed) ? parsed : null;
const entry = arr && arr.length > 0 ? (arr[0] as StatsEntry) : null;
if (!entry) {
return { success: false, warning: false, finalSrtPath: null, cueCount: null, reason: "stats.json has no entries" };
}
if (entry.status !== "success") {
return { success: false, warning: false, finalSrtPath: null, cueCount: null, reason: `stats reports status=${entry.status}` };
}
const errors = entry.metadata?.errors ?? [];
if (Array.isArray(errors) && errors.length > 0) {
return { success: false, warning: false, finalSrtPath: null, cueCount: null, reason: `stats reports ${errors.length} error(s)` };
}
const final = entry.metadata?.output_files?.final_srt ?? null;
if (!final || !fs.existsSync(final)) {
return { success: false, warning: false, finalSrtPath: null, cueCount: null, reason: "final_srt missing from disk" };
}
const cueCount = entry.metadata?.summary?.final_subtitles_refined ?? null;
const warning = cueCount === 0;
return {
success: true,
warning,
finalSrtPath: final,
cueCount,
reason: warning ? "0 cues — likely no speech" : null,
};
}
async function findSingleSrt(dir: string): Promise<string | null> {
let entries: import("node:fs").Dirent[];
try {
entries = await fsp.readdir(dir, { withFileTypes: true });
} catch {
return null;
}
const srts = entries
.filter((e) => e.isFile() && e.name.toLowerCase().endsWith(".srt"))
.map((e) => path.join(dir, e.name));
return srts.length === 1 ? srts[0]! : null;
}
export function newJobId(): string {
return randomUUID();
}
export function jobBaseDir(): string {
return path.join(process.cwd(), "data", "whisperjav-jobs");
}
export function jobDirFor(id: string): string {
return path.join(jobBaseDir(), id);
}
/** Cross-device-aware move. Falls back to copy + unlink when rename
* hits EXDEV (different filesystems / drives). */
export async function moveFile(src: string, dest: string): Promise<void> {
await fsp.mkdir(path.dirname(dest), { recursive: true });
try {
await fsp.rename(src, dest);
return;
} catch (e) {
const code = (e as NodeJS.ErrnoException).code;
if (code !== "EXDEV") throw e;
}
await fsp.copyFile(src, dest);
await fsp.unlink(src).catch(() => { /* best effort */ });
}
/** True if a directory is writable (heuristic — try to create + remove
* a probe file). Used to choose between beside-video output and the
* data-folder fallback. */
export async function isDirWritable(dir: string): Promise<boolean> {
const probe = path.join(dir, `.pinkudex-write-probe-${process.pid}-${Date.now()}`);
try {
await fsp.writeFile(probe, "");
await fsp.unlink(probe).catch(() => { /* ignore */ });
return true;
} catch {
return false;
}
}
+51
View File
@@ -0,0 +1,51 @@
import "server-only";
export type JobStatus =
| "queued"
| "running"
| "completed"
| "warning"
| "failed"
| "cancelled";
export interface JobRow {
id: string;
code: string;
videoAbs: string;
jobDir: string;
/** Final destination of the moved .srt, populated on success/warning. */
targetSubtitlePath: string | null;
status: JobStatus;
enqueuedAt: number;
startedAt: number | null;
endedAt: number | null;
exitCode: number | null;
error: string | null;
/** Latest "Step X/Y: <description>" parsed from stderr. */
stage: string | null;
stageIndex: number | null;
stageTotal: number | null;
cueCount: number | null;
/** JSON-stringified args array passed to whisperjav. */
cliArgs: string;
logPath: string;
statsPath: string | null;
/** Source video duration (sec) at enqueue time — used for ETA. Null
* when the video hasn't been probed yet. */
videoDurationSec: number | null;
/** WhisperJAV --mode at enqueue time. Persisted so historical ETA
* multipliers can be grouped per mode. */
mode: string | null;
}
/** Snapshot returned by the detail endpoint — JobRow plus a tail of log lines. */
export interface JobDetail extends JobRow {
logTail: string[];
}
export interface EnqueueRequest {
code: string;
partIdx: number;
/** When true, allow overwriting an existing target subtitle file. */
overwrite?: boolean;
}