From e029e898e9cda6f838e12068480dab9cf9df10bc Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 22 May 2026 21:39:09 +0200 Subject: [PATCH] Initial snapshot before step 10 package split --- .gitignore | 7 + AGENTS.md | 71 + CLAUDE.md | 1 + CLAUDE.md.bak | 59 + README.md | 233 ++ TODO.md | 9 + config.json | 5 + fixtures/README.md | 83 + fixtures/filename-extraction.json | 24 + fixtures/query-extraction.json | 22 + fixtures/run.py | 70 + fixtures/shared-normalization.json | 17 + mockups/console-consolidation-claude.html | 695 ++++++ mockups/console-consolidation-options.html | 355 ++++ rc-jav.py | 2230 ++++++++++++++++++++ tests/test_rules.py | 74 + 16 files changed, 3955 insertions(+) create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 CLAUDE.md.bak create mode 100644 README.md create mode 100644 TODO.md create mode 100644 config.json create mode 100644 fixtures/README.md create mode 100644 fixtures/filename-extraction.json create mode 100644 fixtures/query-extraction.json create mode 100644 fixtures/run.py create mode 100644 fixtures/shared-normalization.json create mode 100644 mockups/console-consolidation-claude.html create mode 100644 mockups/console-consolidation-options.html create mode 100644 rc-jav.py create mode 100644 tests/test_rules.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..432eb49 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +scan-cancel.flag +cache.json +cache.json.tmp +reports/ +.claude/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..dd71142 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,71 @@ +# rc-jav (Python CLI) + +Session memory for Codex. Read before making changes here. + +## What this is + +A read-only rclone library comparison + search CLI. Compares `cq:JAV` remote (rclone crypt) against itself (dupe detection) or against external WinCatalog CSV/XML exports. Powers the rclone-jav Brave extension via native messaging. + +## Architecture + +``` +rc-jav.py + ├── reads config.json (default_target etc.) + ├── reads cache.json (per-remote file index, written by --scan) + ├── shells out to: rclone lsf / rclone lsjson / rclone size --json + ├── extract_id() per filename → normalized ID with optional #partN / variant suffix + ├── two query modes: --quick (live rclone --include glob) and cached (uses cache.json) + └── output: rich tables (default) | --basic plain | --format json (for extension) +``` + +## Files + +``` +D:\DEV\Project\rclone-jav\ +├── rc-jav.py single-file CLI +├── config.json default_source/target/catalog (user-editable via --save) +├── cache.json scanned remote file index (written by --scan) +├── wincatalog\ drop WinCatalog CSV/XML exports here (auto-loaded) +├── TODO.md deferred work +└── README.md +``` + +## Companion project + +`D:\DEV\Extensions\Production\rclone-jav\` (PC 1) / `D:\DEV\Extensions\Staging\rclone-jav\` (PC 2) — Brave extension + native messaging host that shells out to `rc-jav.py` for searches. + +## ID normalization + +- `extract_id()` chops trailing single letters (e.g. `IBW-902z.mp4` → `IBW-902`). Decision is intentional — see extension's AGENTS.md "Decision log". +- JAV IDs are canonicalized with at least 3 digits (`ABC-27` → `ABC-027`); 4+ digit IDs keep their width (`ABCD-1294`). User expects real JAV IDs to be `ABC-027`, never `ABC-27` or `ABC-0027`. +- Part suffix detection: `_1`, `-pt1`, `(1)` → appended as `#partN` for distinctness. +- Compound prefixes (`FC2-PPV-123`) handled via secondary regex. +- Search matcher does prefix lookup so `IBW-902` finds both `IBW-902` and `IBW-902#part1` etc. +- Quick search must emit only canonical padded uppercase globs (`ABC-027*`, `ABCDE-1167*`). Do not add `--ignore-case`; user never uses lowercase filenames and it caused noticeable delay. + +## Defaults from earlier sessions + +- `cq:JAV` is the current remote root (after the rclone crypt config change moved it down a level) +- `default_target` in config.json = `["cq:JAV"]` +- `human_size()` formats to 2 decimals (e.g. `6.94 GiB`) +- After the 3-digit ID canonicalization change, run `python rc-jav.py --scan` to rebuild `cache.json` under the new padded keys. +- Duplicate KEEP ranking uses configurable VIP folders before source/size/format ranking. Default VIP folder is `ClearJAV`; video files there are treated as the trusted direct-rip copy. +- Duplicate KEEP ranking treats `.ts` as the lowest-priority video container when any non-`.ts` duplicate is available. + +## Recent decisions / bug fixes + +- `--format json` should keep stdout as clean JSON. Status/progress text belongs on stderr in JSON mode. +- Catalog rows are informational. CSV exports mark them as `CATALOG`; JSON exports put them under `catalog`, not `delete_candidates`. +- Cache loading validates the top-level shape and falls back to an empty cache when `remotes` is missing or malformed. +- The old `--recursive/-R` flag was removed because scans are always recursive (`rclone lsf -R` / quick `lsjson -R`). + +## TODO + +See `TODO.md` for deferred work. + +## When making changes + +- Adding CLI flags: also update host invocation in `D:\DEV\Extensions\Production\rclone-jav\host\rcjav-host.py` if the flag matters to the extension +- Changing `extract_id()` semantics: forces a `--scan` to rebuild cache under new keys, and may need a parallel change in extension's `normalizeId()` +- JSON output format changes: extension's popup.js / overlay rendering reads `structured` array — keep field names stable (`source`, `remote`, `path`, `full_path`, `size`, `size_human`, `mod_time`, `jav_id`) +- Config schema: update `--save` writer and any defaults diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CLAUDE.md.bak b/CLAUDE.md.bak new file mode 100644 index 0000000..3b425fe --- /dev/null +++ b/CLAUDE.md.bak @@ -0,0 +1,59 @@ +# rc-jav (Python CLI) + +Session memory for Claude. Read before making changes here. + +## What this is + +A read-only rclone library comparison + search CLI. Compares `cq:JAV` remote (rclone crypt) against itself (dupe detection) or against external WinCatalog CSV/XML exports. Powers the rclone-jav Brave extension via native messaging. + +## Architecture + +``` +rc-jav.py + ├── reads config.json (default_target etc.) + ├── reads cache.json (per-remote file index, written by --scan) + ├── shells out to: rclone lsf / rclone lsjson / rclone size --json + ├── extract_id() per filename → normalized ID with optional #partN / variant suffix + ├── two query modes: --quick (live rclone --include glob) and cached (uses cache.json) + └── output: rich tables (default) | --basic plain | --format json (for extension) +``` + +## Files + +``` +D:\DEV\Project\rclone-jav\ +├── rc-jav.py single-file CLI +├── config.json default_source/target/catalog (user-editable via --save) +├── cache.json scanned remote file index (written by --scan) +├── wincatalog\ drop WinCatalog CSV/XML exports here (auto-loaded) +├── TODO.md deferred work +└── README.md +``` + +## Companion project + +`D:\DEV\Extensions\Production\rclone-jav\` (PC 1) / `D:\DEV\Extensions\Staging\rclone-jav\` (PC 2) — Brave extension + native messaging host that shells out to `rc-jav.py` for searches. + +## ID normalization + +- `extract_id()` chops trailing single letters (e.g. `IBW-902z.mp4` → `IBW-902`). Decision is intentional — see extension's CLAUDE.md "Decision log". +- Part suffix detection: `_1`, `-pt1`, `(1)` → appended as `#partN` for distinctness. +- Compound prefixes (`FC2-PPV-123`) handled via secondary regex. +- Search matcher does prefix lookup so `IBW-902` finds both `IBW-902` and `IBW-902#part1` etc. + +## Defaults from earlier sessions + +- `cq:JAV` is the current remote root (after the rclone crypt config change moved it down a level) +- `default_target` in config.json = `["cq:JAV"]` +- `human_size()` formats to 2 decimals (e.g. `6.94 GiB`) + +## TODO + +See `TODO.md`. Current item: WinCatalog `\` → `/` path normalization in load_catalog_*. + +## When making changes + +- Adding CLI flags: also update host invocation in `D:\DEV\Extensions\Production\rclone-jav\host\rcjav-host.py` if the flag matters to the extension +- Changing `extract_id()` semantics: forces a `--scan` to rebuild cache under new keys, and may need a parallel change in extension's `normalizeId()` +- JSON output format changes: extension's popup.js / overlay rendering reads `structured` array — keep field names stable (`source`, `remote`, `path`, `full_path`, `size`, `size_human`, `mod_time`, `jav_id`) +- Config schema: update `--save` writer and any defaults diff --git a/README.md b/README.md new file mode 100644 index 0000000..08a99d2 --- /dev/null +++ b/README.md @@ -0,0 +1,233 @@ +# rc-jav + +Read-only duplicate scanner for JAV files across rclone remotes. Groups files by JAV ID (e.g. `SSIS-001`) and reports which copy to keep based on priority rules. + +## Priority rules + +1. Video files inside configured **VIP folders** win first. Default VIP folder: `ClearJAV`. +2. If no VIP-folder video exists, **Source always wins** regardless of resolution/size. +3. `.ts` files rank below other video containers, even when the transport-stream copy is larger. +4. If no Source copy exists in the group, **largest file size wins** among the remaining Targets. +5. Suggestions only — script never deletes. Manual cleanup. + +## ID matching + +Filename stem is matched against: + +- Primary: `^([A-Za-z]+)-(\d+)` — `SSIS-001`, `MIDV-123`, `ABP-456` +- Compound: `^(\w+(?:-\w+)+)-(\d+)` — `FC2-PPV-4894535`, `HEYZO-HD-1234` +- Fallback: `^([A-Za-z0-9]+)-(\d+)` — `1pondo-123`, `carib-456` + +IDs normalized to uppercase with leading zeros stripped from the number (so `ssis-001` == `SSIS-1` == `SSIS-001`). Anything after the ID (` - Actress [1080p]`) is ignored for matching. + +### Part-suffix handling + +Multi-part files (`_1`, `_2`, `-1`, `-2`, `_A`, `_B`, `.1of4`, ` (1)`, `-pt1`, `-part1`, `-cd1`, `-disc1`, trailing ` N`) are normalized as `{ID}#partN` so they do not collide as false duplicates. Searching the base ID still finds all parts. Lettered `_A` / `_B` suffixes become part 1 / part 2. + +Add more suffix shapes with repeatable `--part-pattern` regexes. The first capture group is the part number or one part letter and the pattern runs against the filename stem: + +```powershell +python rc-jav.py --scan --part-pattern '[-_ ]side[-_ ]?(A|B)$' +python rc-jav.py --part-pattern '_([CD])$' --save +``` + +Saved rules live in `config.json` as `part_patterns`. The extension Options page has the same custom part detector list for host-triggered searches, duplicate review, and cache rebuilds. + +Files with no parseable ID are listed under "Skipped" at the end so you can spot misnamed files. + +### Rule checks + +Focused rule tests cover ID extraction, multipart grouping safety, and duplicate KEEP ranking: + +```powershell +python -B -m unittest discover -s tests -v +``` + +## Usage + +``` +python rc-jav.py \ + --source cq:personal-files/ClearJAV/ichika-matsumoto \ + --target cq:personal-files/JAV/TMP \ +``` + +Flags: +- `--source` / `-s REMOTE` — priority remote path. Repeat for multiple. +- `--target` / `-t REMOTE` — non-priority remote path. Repeat for multiple. +- `--format {console,txt,csv,json,all}` — default `console`. Non-console formats write to `--output-dir`. +- `--output-dir DIR` — default `./reports`. +- `--no-color` — disable ANSI colors but keep rich layout (tables, panels). +- `--basic` — plain text output, no rich tables/panels. Progress ticks every 25 files on stderr. Useful for piping or simple terminals. +- `--rclone-bin PATH` — path to rclone executable (default: `rclone` on PATH). Example: `--rclone-bin C:\Programs\rclone\rclone.exe`. +- `--clearjav` — shortcut: sets source = `DEFAULT_SOURCE`, target = `DEFAULT_TARGET`. Equivalent to `--source cq:personal-files/ClearJAV --target cq:personal-files/JAV/TMP`. Combine with `--source`/`--target` to override one side. + +Examples: + +``` +# full library dupe scan, one flag +python rc-jav.py --clearjav + +# same but only check one actress folder against TMP +python rc-jav.py --clearjav --source cq:personal-files/ClearJAV/ichika-matsumoto +``` + +## Search mode + +Check whether a JAV ID already exists in your library before downloading: + +``` +python rc-jav.py --search SSIS-001 +python rc-jav.py --search SSIS-001 --search FC2-PPV-4894535 + +# wildcards (quote to avoid shell glob expansion) +python rc-jav.py --search "IPZZ-*" +python rc-jav.py --search "FC2-PPV-*" +python rc-jav.py --search "SSIS-???" # exact 3-digit numeric +``` + +Wildcard syntax: `*` (any chars) and `?` (one char), case-insensitive. Matches against normalized IDs in the index, including `#partN` suffixes automatically. + +Range syntax: `[N-M]` inclusive both ends. Works inside any prefix. + +``` +python rc-jav.py --search "IPZZ-[820-860]" +python rc-jav.py --search "FC2-PPV-[4894500-4894600]" +python rc-jav.py --search "MIDV-[001-010]" # zero-padding preserved +``` + +Quote in PowerShell/bash so `[...]` reaches Python literally. Reversed ranges (`860-820`) auto-swap. + +With no `--source` / `--target` flags, only `DEFAULT_TARGET` (TMP) is scanned — the typical case for "do I already have this in my unsorted pile?". Pass `--source cq:personal-files/ClearJAV` to also check the priority library. Edit `DEFAULT_SOURCE` / `DEFAULT_TARGET` at the top of the script to change defaults. Remote scans are recursive. + +Exit code: `0` if every query had at least one hit, `1` otherwise — useful for shell automation. + +## Name search (`--name`) + +Substring search against filenames (case-insensitive). Find all files by actress, studio, tag, anything that appears in the filename. + +``` +python rc-jav.py --name Ichika +python rc-jav.py --name "Ichika Matsumoto" +python rc-jav.py --name Ichika --name Yui # OR — files matching either +python rc-jav.py --name "Mat*" # glob wildcard +python rc-jav.py --search IPZZ-860 --name Ichika # both — separate result blocks +``` + +- Multiple `--name` tokens = OR. Use one combined `--name "foo bar"` for AND/exact-substring. +- Matches against the filename stem only (not folder names). +- Auto-routes to **cached** mode because substring globs can't be server-side filtered on most backends. Pass `-q` to force quick anyway (slower). + +### Smart search mode (auto quick / cached) + +The script auto-picks the right execution path per query and prints which one it chose: + +| Query shape | Picked mode | Reason | +|---|---|---| +| Single exact ID (`IPZZ-860`) | quick | live rclone `--include`, ~1–2s even on huge trees | +| Wildcard (`IPZZ-*`, `SSIS-???`) | cached | reliable normalized matching | +| Range (`IPZZ-[820-860]`) | cached | avoids N rclone calls | +| Multiple `--search` flags | cached | warmup amortizes | + +Override: +- `--quick` / `-q` — force live rclone lookup (skips cache). +- `--cache` — force cache (builds it if cold). + +Quick mode never reads or writes the cache. Cache mode honors `--update` and `--no-cache` as before. + +### Cache + +Search mode caches each remote's file list in `./cache.json` next to the script. Subsequent searches are near-instant. + +- First run: scans + writes cache. +- Later runs: reads cache (banner shows `CACHED 14m (154 files)`). +- `--update` / `-u`: force re-scan + overwrite cache for the requested remotes. +- `--no-cache`: bypass cache (no read, no write). +- Stale warning when cache is older than 24h — still used, marked `CACHED-STALE`. +- Ctrl+C during a scan: rclone is terminated, cache for in-flight remote is NOT written. + +Delete `cache.json` to reset everything. + +### Saving defaults (--save) + +Persist `--source`, `--target`, `--catalog`, and/or `--part-pattern` to `config.json` so you don't have to type them every run. + +``` +# set default target +python rc-jav.py --target cq:personal-files/JAV/TMP --save + +# set source + multiple targets at once +python rc-jav.py --source cq:personal-files/ClearJAV ^ + --target cq:personal-files/JAV/TMP ^ + --target cq:personal-files/JAV/SORTED ^ + --save + +# inspect +type config.json +``` + +Only the keys you explicitly pass are written — running `--save --target X` won't wipe a saved `default_source`. Delete `config.json` to reset to the hardcoded defaults at the top of `rc-jav.py`. + +### Scan-only (--scan) + +Refresh the cache without running a search or dupe report — useful for Task Scheduler / cron pre-warming. + +``` +# default: refresh DEFAULT_TARGET (TMP) +python rc-jav.py --scan + +# refresh both source and target +python rc-jav.py --scan --source cq:personal-files/ClearJAV --target cq:personal-files/JAV/TMP + +# nightly via Task Scheduler +schtasks /Create /SC DAILY /ST 03:00 /TN "rc-jav nightly scan" ^ + /TR "python D:\DEV\Project\rclone-jav\rc-jav.py --scan --basic" +``` + +`--scan` always overwrites the cache for the remotes you list. Exit 0 = success, non-zero = rclone failure. + +``` +python rc-jav.py --search MIDV-999 ; if ($LASTEXITCODE -eq 0) { "have it" } else { "download" } +``` + +## WinCatalog integration + +WinCatalog's native `.wcat` format is proprietary, so the script reads its exports instead. + +1. In WinCatalog: **File → Export** → choose **CSV** or **XML**. +2. Save into the `wincatalog/` folder next to the script. All `*.csv` and `*.xml` files there are auto-loaded — drop in as many discs as you want. +3. Run as normal: `python rc-jav.py --search IPZZ-860` +4. Override or add extra paths with `--catalog PATH` (file or folder, repeatable). +5. To change the default folder, edit `DEFAULT_CATALOG` at the top of the script. + +Re-export when your catalog changes; the script re-reads on every run (catalog data is **not** cached — it's already a local file). + +**Role of catalog hits:** +- Search: shown as rows with source label `Catalog`. The disc/volume name is encoded into the path so you know which offline backup holds the file. +- Dupe mode: catalog entries appear in groups for awareness but are **never marked KEEP or DELETE?** — they're offline, can't be touched. A group is only flagged as a dupe when 2+ rclone copies exist. + +**CSV column auto-detection** (case-insensitive, first match wins): +- Name: `Name`, `File Name`, `Filename`, `Title` +- Path: `Path`, `Full Path`, `Location`, `Folder` +- Size: `Size`, `File Size`, `Bytes`, `Size (bytes)` +- Disc: `Disc`, `Disc Name`, `Disc Label`, `Volume`, `Source`, `Catalog`, `Media` + +XML: walks the tree, treats `` / `` nodes inside `` / `` / `` containers, with `` nesting. + +## Requirements + +- Python 3.9+ +- `pip install rich` (used for progress bars + themed output) +- `rclone` on `PATH` with the relevant remotes configured. + +## UI + +- Live per-file progress bar during scans (`rclone size --json` for total, then `rclone lsf --files-only -R --format pst` streamed). +- Banner panel showing run mode + per-remote cache status. +- Rich tables for search hits and duplicate groups. +- `--no-color` for plain output (CI, piping). + +## Roadmap + +- Phase 1 (current): report duplicates + search. +- Phase 2: `--apply` mode that runs `rclone delete` on `DELETE?` candidates behind a confirmation gate. +- Phase 3: resolution-aware tiebreakers, move-to-review folder, scheduled runs. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..f831295 --- /dev/null +++ b/TODO.md @@ -0,0 +1,9 @@ +# TODO / Deferred work + +## Deferred + +(append below) + +## Completed notes + +- WinCatalog CSV/XML paths are normalized from `\` to `/` during catalog load. diff --git a/config.json b/config.json new file mode 100644 index 0000000..496030f --- /dev/null +++ b/config.json @@ -0,0 +1,5 @@ +{ + "default_target": [ + "cq:JAV" + ] +} \ No newline at end of file diff --git a/fixtures/README.md b/fixtures/README.md new file mode 100644 index 0000000..2a00990 --- /dev/null +++ b/fixtures/README.md @@ -0,0 +1,83 @@ +# Shared JAV ID fixture corpus + +JSON cases shared between the Python `rc-jav.py` CLI and the browser +extension at `D:\DEV\Extensions\Production\rclone-jav\`. Each side +reads the cases relevant to its own extraction surface. + +## Files + +| File | Domain | Consumer | Notes | +|-------------------------------|----------|----------------------------------------|-------| +| `filename-extraction.json` | filename | Python `extract_id(name)` | Has `#partN` expectations for multipart files | +| `query-extraction.json` | query | Extension `content.js` `normalizeId` | Looser context; extension never emits part suffix | +| `shared-normalization.json` | shared | BOTH | Contract: any mismatch here is a bug, not a fixture issue | + +All files share the same shape: + +```json +{ + "version": 1, + "domain": "…", + "description": "…", + "case_schema": { … }, + "cases": [ + { "name": "…", "input": "…", "expected": "…" } + ] +} +``` + +`expected: null` means "no ID should be detected". + +## Running the Python side + +```bash +python fixtures/run.py +``` + +The runner imports `rc-jav.py` in place, exercises `extract_id` against +`filename-extraction.json`, and `normalize_id` against +`shared-normalization.json`. Exit code is non-zero on any failure. + +## Running the extension side + +No automated runner today. `content.js` lives inside an IIFE that the +browser injects into pages, so importing it from Node would require +either an extraction refactor or a duplicated copy of the regex. Until +that lands, treat `query-extraction.json` and `shared-normalization.json` +as the canonical specification: if you touch `ID_RE_DASHED`, +`ID_RE_UNDASHED`, or `BUILTIN_ID_NORMALIZERS` in content.js, eyeball +this corpus and confirm the cases still describe expected behavior. + +## Adding a case + +1. Pick the file matching the surface you're testing. +2. Append a `{ "name", "input", "expected" }` entry. Keep `name` + descriptive — it's the only label shown when the runner fails. +3. If the case exercises a guarantee both sides must honor, add it to + `shared-normalization.json` as well. +4. Run `python fixtures/run.py` to confirm Python still passes. + +## Known cross-side divergences (intentional) + +These are NOT bugs — they reflect the different surfaces each side +extracts from. Recorded here so future contributors don't try to +"fix" them. + +- **`FC2PPV1841460` compact form (no dashes).** The extension's + `BUILTIN_ID_NORMALIZERS` in `content.js` rewrites this to + `FC2-PPV-1841460` when seen in page titles. Python `extract_id` + does NOT — the compact form doesn't realistically appear in + filenames on disk. Hence the case lives in + `query-extraction.json` only, not in `filename-extraction.json` or + `shared-normalization.json`. + +If a case belongs to one side's contract but not the other's, file it +under the specific domain (`filename-` or `query-`) — not under +`shared-`. + +## Ownership + +This directory lives in the Python repo only because the Python repo +is the more stable root. Conceptually it's joint property of both +codebases. Don't add anything Python-specific to the JSON files — keep +them tool-neutral. diff --git a/fixtures/filename-extraction.json b/fixtures/filename-extraction.json new file mode 100644 index 0000000..fc656b6 --- /dev/null +++ b/fixtures/filename-extraction.json @@ -0,0 +1,24 @@ +{ + "version": 1, + "domain": "filename", + "description": "Filename → canonical JAV ID (with optional #partN suffix). Consumed by Python rc-jav.extract_id.", + "case_schema": { + "name": "human label", + "input": "filename including extension", + "expected": "canonical ID (e.g. ABC-001 or ABC-001#part1) or null when no ID present" + }, + "cases": [ + { "name": "plain dashed ID", "input": "ABC-027.mp4", "expected": "ABC-027" }, + { "name": "dashed ID with resolution tag", "input": "SCOP-297 [1080p].mp4", "expected": "SCOP-297" }, + { "name": "bracket-wrapped ID", "input": "[REAL-779].mp4", "expected": "REAL-779" }, + { "name": "bracket-wrapped ID with extra tag", "input": "[SCOP-297] [1080p].mp4", "expected": "SCOP-297" }, + { "name": "no-hyphen fallback", "input": "MVSD312.avi", "expected": "MVSD-312" }, + { "name": "trailing lowercase variant letter", "input": "IBW-902z.mp4", "expected": "IBW-902z" }, + { "name": "multipart _PART suffix", "input": "KV-118 - Aiba Reika_PART1.mp4", "expected": "KV-118#part1" }, + { "name": "multipart _A letter suffix", "input": "KV-118_A.mp4", "expected": "KV-118#part1" }, + { "name": "multipart trailing -N before bracket", "input": "OFJE-195-7 [480p].mp4", "expected": "OFJE-195#part7" }, + { "name": "FC2 PPV plain", "input": "FC2-1841460.mp4", "expected": "FC2-PPV-1841460" }, + { "name": "FC2 PPV explicit", "input": "FC2-PPV-1841460.mp4", "expected": "FC2-PPV-1841460" }, + { "name": "no ID present", "input": "random_video.mp4", "expected": null } + ] +} diff --git a/fixtures/query-extraction.json b/fixtures/query-extraction.json new file mode 100644 index 0000000..25cdcce --- /dev/null +++ b/fixtures/query-extraction.json @@ -0,0 +1,22 @@ +{ + "version": 1, + "domain": "query", + "description": "Page text / title -> canonical JAV ID. Consumed by the browser extension (content.js normalizeId). Difference from filename: looser context (sentences, mixed punctuation, site chrome). Includes forms (e.g. FC2PPV compact) that Python extract_id does NOT handle, by design — see fixtures/README.md.", + "case_schema": { + "name": "human label", + "input": "raw page text", + "expected": "canonical ID without part suffix (extension never emits #partN), or null when no ID found" + }, + "cases": [ + { "name": "title with site chrome", "input": "SSIS-001 — JAV.tube", "expected": "SSIS-001" }, + { "name": "title with description", "input": "Watch SSIS-001 1080p HD Online", "expected": "SSIS-001" }, + { "name": "trailing letter variant", "input": "IBW-902z Full Movie", "expected": "IBW-902" }, + { "name": "no hyphen in title", "input": "MVSD312 stream", "expected": "MVSD-312" }, + { "name": "FC2 PPV compact", "input": "FC2PPV-1841460 — preview", "expected": "FC2-PPV-1841460" }, + { "name": "FC2 plain digits", "input": "FC2-1841460 thumbnail", "expected": "FC2-PPV-1841460" }, + { "name": "FC2-PPV explicit", "input": "FC2-PPV-1841460 Full", "expected": "FC2-PPV-1841460" }, + { "name": "leading zeros preserved", "input": "ABF-042 — sample", "expected": "ABF-042" }, + { "name": "long numeric tail (7 digits)", "input": "BLK-4748520 stream", "expected": "BLK-4748520" }, + { "name": "no ID present", "input": "JAV Database · home", "expected": null } + ] +} diff --git a/fixtures/run.py b/fixtures/run.py new file mode 100644 index 0000000..3f97e68 --- /dev/null +++ b/fixtures/run.py @@ -0,0 +1,70 @@ +"""Run the shared JAV-ID fixture corpus against rc-jav.py. + +Exits non-zero if any fixture case fails. No third-party dependencies. + +Usage: + python fixtures/run.py +""" +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = Path(__file__).resolve().parent + +SPEC = importlib.util.spec_from_file_location("rcjav", ROOT / "rc-jav.py") +RCJAV = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = RCJAV +SPEC.loader.exec_module(RCJAV) + + +def _load(name: str) -> dict: + with (FIXTURES / name).open("r", encoding="utf-8") as f: + return json.load(f) + + +def _run(label: str, cases: list[dict], fn) -> tuple[int, int]: + passed = 0 + failed = 0 + for case in cases: + got = fn(case["input"]) + if got == case["expected"]: + passed += 1 + else: + failed += 1 + print(f" FAIL [{label}] {case['name']!r}") + print(f" input = {case['input']!r}") + print(f" expected = {case['expected']!r}") + print(f" got = {got!r}") + return passed, failed + + +def main() -> int: + total_passed = 0 + total_failed = 0 + + for filename, fn_name, fn in [ + ("filename-extraction.json", "extract_id", RCJAV.extract_id), + ("shared-normalization.json", "normalize_id", RCJAV.normalize_id), + ]: + doc = _load(filename) + cases = doc.get("cases", []) + print(f"\n{filename} -> rcjav.{fn_name} ({len(cases)} cases)") + p, f = _run(filename, cases, fn) + total_passed += p + total_failed += f + print(f" {p} passed | {f} failed") + + print() + if total_failed: + print(f"FAILED: {total_failed} of {total_passed + total_failed} cases") + return 1 + print(f"OK: all {total_passed} cases passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fixtures/shared-normalization.json b/fixtures/shared-normalization.json new file mode 100644 index 0000000..7880f5b --- /dev/null +++ b/fixtures/shared-normalization.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "domain": "shared", + "description": "Raw ID forms → canonical form. Both Python (normalize_id) and the extension (content.js normalizeId) MUST agree on these. Mismatch here is a contract bug.", + "case_schema": { + "name": "human label", + "input": "raw ID-bearing token (no path, no extension)", + "expected": "canonical ID" + }, + "cases": [ + { "name": "lowercase prefix uppercased", "input": "abc-027", "expected": "ABC-027" }, + { "name": "FC2 plain -> FC2-PPV", "input": "FC2-1841460", "expected": "FC2-PPV-1841460" }, + { "name": "FC2-PPV explicit preserved", "input": "FC2-PPV-1841460", "expected": "FC2-PPV-1841460" }, + { "name": "leading zeros preserved", "input": "ABF-042", "expected": "ABF-042" }, + { "name": "5-digit numeric segment", "input": "SDDE-12345", "expected": "SDDE-12345" } + ] +} diff --git a/mockups/console-consolidation-claude.html b/mockups/console-consolidation-claude.html new file mode 100644 index 0000000..05ed4c8 --- /dev/null +++ b/mockups/console-consolidation-claude.html @@ -0,0 +1,695 @@ + + + + + rclone-jav consolidation — final converged plan + + + +
+

rclone-jav Consolidation — Final Converged Plan

+
+ + Status: execution in progress. Shipped: steps 1 (Sim Dupe delete), 2 (CSS extraction), 3 (Transfer Assistant delete + Diagnostics replacement), 5 (Recent Activity + Search Troubleshooting → new Debug Tools pane). Pending: steps 6 (options.js split — Cache & Dup Review paired, biggest), 7a (Bulk Check standalone window), 8 (fixtures), 9 (cache contract), 10 (rc-jav.py split), 11 (host fast-path decision). See D:\DEV\Extensions\Production\rclone-jav\AGENTS.md "Console consolidation refactor — execution status" for current state. +
+ +
+
+

✓ Decided

+
    +
  • Console / Settings / Support tri-split
    • +
  • Default landing = Duplicate Review
    • +
  • Status badges on tabs, no dashboard pane
    • +
  • Launcher pattern over toolbox
    • +
  • Keep Ranking nested in Dup Review
    • +
  • Sim Dupe → delete, samples/ HTML harness
    • +
  • Transfer wizard → delete after Diagnostics replacement verified
    • +
  • Bulk ID Check → detached chrome.windows popup, NOT a Console sidebar tab
    • +
  • Inline rule tests stay, standalone benches → Debug
    • +
+
+
+

✓ Shipped

+
    +
  • Step 1: Sim Dupe deleted from popup. samples/sim-dupe.js preserves payload.
    • +
  • Step 2: CSS extracted → options.css. options.html 1179 → 794 lines.
    • +
  • Step 3: Transfer wizard deleted. Diagnostics → Native host registration now shows Extension ID + Copy button.
    • +
  • Step 5: Recent Activity + Search Troubleshooting moved to new Debug Tools pane. Scope verified by code read.
    • +
+
+
+

📋 Pending

+
    +
  • Step 6: options.js split (Cache + Dup Review paired). 3133-line file. Biggest, riskiest.
    • +
  • Step 7a: bulk-check.html standalone + popup launcher.
    • +
  • Steps 8–10: fixtures, cache contract, rc-jav.py split.
    • +
  • Step 11: host fast-path benchmark + narrow/delete decision.
    • +
+
+
+ + +

1. Primary recommended layout

+

Default landing = Duplicate Review (user's most-frequent maintenance workflow). Sidebar tab labels carry live status badges — no dashboard pane needed. Launcher pattern: heavy tools open focused panes, not nested fieldsets.

+ +
+
+ rclone-jav +
+ + +
+
+
+ +
+
+
+

Duplicate Review 27 pending

+

After-upload workflow. Risky groups skipped by default. Keep Ranking lives here as configuration, not in a separate Settings tab.

+
+
+ + +
+
+ +
+ Pending Review + Skipped — Risky + Keep Ranking Rules + Delete History +
+ +
+
+

Filter (this tool only)

+
+ + + +
+

Filters scoped — never exported as global settings.

+
+
+

Delete queue

+
12 files · 47.3 GiB
+

Safety: VIP folders + multipart-risk paths auto-excluded.

+
+
+ +
+

JBD-291 · 2 candidates

+
+
+ /JAV/clearjav/JBD-291 [1080p].mp4 + 4.94 GiB + KEEP +
+
+ /JAV/old/JBD-291.mp4 + 3.82 GiB + +
+
+
+ +
+

OFJE-195 · multipart risk

+
+
+ /JAV/OFJE-195_PART1.mp4 + 2.10 GiB + REVIEW +
+
+ /JAV/OFJE-195_PART2.mp4 + 2.08 GiB + REVIEW +
+
+
+
+
+
+ + +

2. Decision table (refactor spec data)

+

Each current pane mapped to its future home, treatment, ship order, and replacement work (if any). Ship order = execution sequence within phase 3 (UI consolidation). Steps share PR scope where useful.

+ +
+ KEEP visible + CONTEXTUAL (lives with feature) + MOVE (relocate) + DEBUG only + DELETE +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Current surfaceFuture homeTreatmentReplacement
1Sim Dupe popup actionsamples/popup-states.html (repo file)DELETENo product replacement. Repo HTML for layout testing only.
2CSS embedded in options.htmlPer-pane .css files alongside per-pane JSEXTRACTNo behavior change. Reduces options.html before JS split.
3Transfer Assistant wizard(gone)DELETEReplacement = Diagnostics 3 actions (see §3 checklist). Delete after verification.
4Cache & Scans paneConsole → Cache & ScansKEEPPaired with Dup Review extraction. State interface shared.
4Duplicate Review paneConsole → Duplicate Review (default landing)KEEPSame PR as Cache & Scans. Reads cache state.
4Keep Ranking Rules paneDuplicate Review → Keep Ranking Rules (sub-tab)CONTEXTUALMoves with Dup Review. Becomes nested sub-tab.
4VIP folders configDuplicate Review → Keep Ranking RulesCONTEXTUALFeature-specific config moves with feature.
5Recent Activity (search/page history)Support → Debug Tools → Search ActivityDEBUGIf audit deletion events also present, split out (pending verification).
5Search TroubleshootingSupport → Debug ToolsDEBUGStandalone bench. No edit locality.
5Page Extraction Test (standalone)Support → Debug ToolsDEBUGInline "Pick Element" variant stays in Site Extraction settings.
5Test ID Extraction (inline)Settings → Matching Rules (collapsible per editor)CONTEXTUALEditor feedback. Stays beside rule it tests.
5Test ID Extraction (standalone bench)Support → Debug ToolsDEBUGSecond row — split from inline version above.
6Library Issues paneConsole → Library IssuesKEEPOwn tab + status badge. Rename UI nested as sub-tab.
6Bulk ID Checkbulk-check.html — detached chrome.windows popupRESHAPERemoved from Console sidebar. Single entry path = popup launcher button → opens 640×540 detached window. Different tool type than Console panes (transient utility, no sidebar context).
7Profiles, Scan Behavior, Overlays, Deletion settingsSettings → (separate sub-tabs)KEEPSettings sub-tabs split into separate JS files.
7Matching Rules / Site ExtractionSettings → (separate sub-tabs, inline tests retained)KEEPCollapsible inline tester beside each rule.
8(new) Shared fixture corpusTop-level fixtures/ (neutral location)NEWContract between extension and Python. Both consume.
9Cache contract designCACHE_VERSION (exists) + ID_RULES_VERSION (new)NEWSchema bump = force rebuild. Rules bump = warn-and-mark-stale.
10rc-jav.py monolithrcjav/ package (ids, cache, dupes, catalog, …)SPLITAfter fixtures + tests + cache contract exist.
11Host fast-path searchNarrow / Delete / Keep — based on §4 benchmarkDECIDEBenchmark idle + under-scan latency first.
+ + +

3. Pre-execution checklists (user handoffs)

+ +
+
+

Diagnostics replacement verification (gates step 3 — Transfer wizard delete)

+
Current extension ID shown as one-line text with copy-to-clipboard button
Replaces wizard's "your extension ID is…" step.
user opens
Diagnostics
+
Button labeled "Re-register host" that triggers register-host.bat path
Replaces wizard's "run this script" step.
user opens
Diagnostics
+
Verification result shown inline within 2s of register click
Replaces wizard's "now check the result" step.
user opens
Diagnostics
+
All three above visible without expanding collapsed sections (one screen)
If buried in expandable cards, write better UI first.
visual
inspection
+
+ +
+

Recent Activity scope test (settles split question)

+
Open Recent Activity. Note current entry types visible.
LIVE search, CACHE search, MATCH, NO_MATCH, NO_ID, page-check, etc.
user
+
Perform a delete in Duplicate Review. Refresh Recent Activity.
Single delete operation, any candidate.
user
+
If delete event appears → audit value exists. Split into Dup Review → Delete History.
If no → single role. Move entire log to Debug Tools.
user reports
+
+
+ +

Cosmetic remaining: popup launcher button label "Bulk Check" vs icon-only. Either works. Default to label until popup row gets crowded.

+ + +

4. Bulk Check — detached window pattern

+

User clarified: Bulk Check is a transient utility, not a persistent Console surface. Doesn't fit sidebar-tab pattern alongside Dup Review / Cache & Scans / Library Issues. Decision: standalone bulk-check.html opened as detached chrome.windows popup, no Console sidebar entry. Single canonical entry path = popup launcher button.

+ +
+
+

Browser-action popup with launcher

+
+
+
+ rclone-jav + +
+ +
+ + + +
+
Single-search stays primary. Bulk button = one click to detached window.
+
+
+
+ +
+

Detached window (640×540) after launch

+
+
+
+ Bulk ID Check +
+ + + +
+
+
+ +
+ + + +
+
+ 5 IDs checked · 3 match · 2 no match +
+
+
BLK-4744.94 GiBMATCH
+
KV-118multi-partMATCH
+
PRTD-029—NO MATCH
+
+
+
+
+

Detached window. No tab bar, no address bar. Closes cleanly when done. Sits over browser, stays visible across tab switches.

+
+
+ +
+

Why detached window, not Console tab

+
+
+

Other Console tools (Dup Review, Cache & Scans, Library Issues):

+
    +
  • long workflows, multi-pass
    • +
  • need sidebar context (compare to other tools)
    • +
  • persistent state (review queue, scan job)
    • +
  • fit Options sidebar tab pattern
    • +
+
+
+

Bulk Check:

+
    +
  • short workflow, one-shot
    • +
  • no sidebar context needed
    • +
  • transient state (last-paste persisted, results ephemeral)
    • +
  • fits detached-window pattern
    • +
+
+
+

Different tool type. Treating it like Dup Review was a category error. Single user knows the feature exists — discovery via popup button is enough.

+
+ +
+

Implementation notes

+ 
// Popup launcher click handler
+chrome.windows.create({
+  url: chrome.runtime.getURL('bulk-check.html'),
+  type: 'popup',
+  width: 640,
+  height: 540
+});
+
    +
  • Window dedup: track open bulk-check window ID in chrome.storage.session. Second launcher click focuses existing window instead of spawning duplicate.
    • +
  • State persistence: last paste saved to chrome.storage.local key bulk_check_last_paste. Reopen restores. Results are ephemeral (re-run on reopen).
    • +
  • Backend reuse: calls native host via same messaging path popup search uses. No new backend code.
    • +
  • No back nav: window can't navigate. User closes when done. Ctrl+W closes the bulk window, not a browser tab.
    • +
+
+ +
+

Edge cases

+
    +
  • Popup auto-closes after launcher click (Chrome behavior). Window survives. Good — that's the intent.
    • +
  • Window positioning unreliable. Chrome treats left/top as hints, multi-monitor users may get the window on the wrong screen. Acceptable for personal-use tool.
    • +
  • Brave / Edge variance. Detached popups behave slightly differently across Chromium forks. Test on user's actual browser before shipping. Fallback if broken: open bulk-check.html in a normal tab via chrome.tabs.create.
    • +
+
+ +
+

Does NOT generalize

+

Detached-window pattern fits Bulk Check because it's transient + no-sidebar-context + short. Doesn't apply to:

+
    +
  • Diagnostics — reference info, lives in sidebar fine
    • +
  • Setup repair button — already inline in Diagnostics, small enough
    • +
  • Dup Review / Cache & Scans / Library Issues — long workflows, sidebar context useful
    • +
  • Settings — set-and-forget, not workflow
    • +
+

One-tool answer, not a pattern across the app.

+
+ + +

5. Execution sequence (final)

+

Codex's revised order (triage first, boundary doc second) with my refinements. Risk and dependencies marked. Steps 1–4 are reversible single-file changes (warmup phase). Steps 5–10 = structural. Step 11 = final architectural call.

+ +
+
1
Per-pane triage — 30 min with user. Decision table above IS this artifact.
zero riskno deps
+
2
Boundary ownership doc — extension extracts query ID, Python owns filename semantics, host adapts. 1 hour, no code.
zero riskafter #1
+
3
Host fast-path benchmark — latency under idle Python AND under scanning Python. Result gates step 11.
measure onlyno deps
+
4
Delete confirmed surfaces — Sim Dupe popup button (no replacement), Transfer wizard (after Diagnostics verification passes).
trivialafter §3 checklist
+
5
CSS extraction from options.html — per-pane CSS files. No behavior change. Bisect-friendly.
lowafter #4
+
6
options.js split: Cache & Dup Review paired — Dup Review reads cache state. Single PR extracts both. Keep Ranking moves with Dup Review.
moderateafter #5
+
7
options.js split: Debug Tools + Library Issues + Settings sub-tabs — remaining Options extractions. Inline test components reused across rule editors. Bulk Check is NOT here — it's a new standalone file (step 7a).
moderateafter #6
+
7a
Create bulk-check.html standalone + popup launcher button — new HTML file, own JS module, no Options dependency. Popup gets one button calling chrome.windows.create. Window dedup + state persistence in chrome.storage.
additiveparallel to #7
+
8
Shared fixture corpus — top-level fixtures/ (neutral). Python and extension both consume.
additiveno blocking
+
9
Cache contract design — CACHE_VERSION (exists) + ID_RULES_VERSION (new). Schema vs semantics, two concepts.
design decisionbefore #10
+
10
rc-jav.py module split — ids.py, cache.py, dupes.py, catalog.py, rclone_io.py, cli.py. Tests pre-exist via #8.
code churnafter #8, #9
+
11
Host narrow / keep / delete — based on #3 benchmark. If under-scan responsiveness depends on host = keep narrow. If not = delete.
behavior changeafter #3, #10
+
+ + +

6. Acceptance criteria template

+

Each step in the sequence needs three things in the final spec: acceptance criterion, rollback procedure, touched-files list. Without these, "ship order N" is a wish not a plan. Template below — fill per step in spec doc.

+ +
+

Template per sequence step

+
step: 6 +title: options.js split — Cache & Dup Review paired + +touched_files: + - options.html (script tag order changes) + - options.js (DELETE: cache section, dup review section, keep ranking section) + - options-cache.js (NEW) + - options-review.js (NEW) + - options-core.js (NEW: shared helpers, pane nav, save/load) + +acceptance: + - Fresh extension reload, options.html opens + - Default landing = Duplicate Review tab + - Cache & Scans tab loads, shows last scan timestamp + - Run Duplicate Review on existing cache — same result set as pre-refactor + - Keep Ranking Rules sub-tab inside Dup Review opens + - No console errors on load or interaction + +rollback: + - git revert <sha> + - No data migration. Cache schema unchanged. Storage keys unchanged. + - Diagnostics-verified replacement of Transfer wizard remains intact (step 3 already shipped).
+
+ + +

7. Out of scope (explicitly rejected)

+
    +
  • Dashboard pane — tab badges replace. Adding a dashboard creates a feature sink.
    • +
  • After-Upload workflow wizard page — sidebar nav order already encodes the workflow.
    • +
  • Matching Lab consolidation page — inline tests cover editor needs, standalone bench in Debug covers diagnostic needs.
    • +
  • Mode switcher top bar (Console / Settings / Support segmented control) — sidebar groups do this.
    • +
  • In-extension Sim Dupe / Debug Preview page — repo HTML file is enough for single-user layout work.
    • +
  • Popup bulk mode toggle — popup stays single-job. Launcher button opens detached window, no inline bulk mode.
    • +
  • Bulk ID Check as Console sidebar tab — wrong tool type for sidebar pattern. Detached window matches its transient nature.
    • +
  • Bulk Check as Options-page deep-link tab — previously considered. Rejected: leaves a leftover tab open after use, Options sidebar adds noise to a one-shot tool.
    • +
  • Frontend framework (React/Vue/Svelte) — vanilla + ordered script files is correct for MV3 + project scale.
    • +
  • Console.log telemetry for usage audit — manual triage of single-user project beats instrumented signals.
    • +
+ +

8. Net position

+

Architecture decided. Three small user handoffs remain (Diagnostics verification, Recent Activity scope check, popup button label). After those, decision table expands into per-step spec with acceptance + rollback. Code work begins on step 1 (smallest, fastest, lowest risk). Total estimated execution span: phased over multiple PRs, no big-bang refactor.

+ +
+ + diff --git a/mockups/console-consolidation-options.html b/mockups/console-consolidation-options.html new file mode 100644 index 0000000..abe511e --- /dev/null +++ b/mockups/console-consolidation-options.html @@ -0,0 +1,355 @@ + + + + + rclone-jav console consolidation direction + + + +
+

rclone-jav Consolidation Direction

+

Updated after the refactor discussion. The page keeps the earlier visual samples, but the decisions are now explicit: launcher-style maintenance console, Duplicate Review as the default work surface, status on navigation instead of a dashboard pane, Bulk ID Check with a popup-launched quick window plus a full Console tool, and debug/testing surfaces pulled out of the daily workflow.

+
+ Frequent maintenance + Set-and-forget settings + Support / debug + Placement decision +
+ +
+
+

1. Recommended Console Shell

Frequent maintenance tools get focused destinations. The navigation carries status instead of a separate dashboard pane.

chosen direction
+
+
rclone-jav Console
+
+ +
+

Duplicate Review

Default landing after uploads. Keep Ranking Rules and delete history stay with the workflow that uses them.

27 pending
+
+

Review Queue

12 ready · 2 risky

Uses VIP folders, multipart safety, keep reasons, and delete queue checks.

+

Contextual Config

Keep Ranking Rules live inside Duplicate Review, not as a distant general setting.

+
+
+

Cache Status Lives In Nav

Cache & Scans owns the scan detail. The sidebar badge is enough while you review dupes.

+

Console Neighbors

Library Issues and Bulk ID Check remain direct tools, not settings fieldsets.

+
+
+
+
+

Chosen: Console / Settings / Support sidebar, Duplicate Review first, no dashboard pane.

+
+ +
+

2. Launcher Treatment

The page should open focused maintenance views instead of collecting every heavy tool as a permanent fieldset.

chosen pattern
+
+
Library Console
+
+
+

Cache Status

Fresh cache · cq:JAV · last scan 28m ago
+

Next Maintenance

Large tools stay out of the page until you open them.

+
+
+
Example focused tool opened from a launcher
+
+

Duplicate Review

Full space for filters, keep reasons, and delete queue safety.

+
2 risky groups skipped by default
+
+
JBD-291 ClearJAV vs older MP4KEEP
+
OFJE-195 multipart warningREVIEW
+
+
+
+
+
+

Chosen for the big surfaces: Duplicate Review, Cache & Scans, Library Issues, and Bulk ID Check.

+
+ +
+

3. Boundary Without A Mode Switcher

Console, Settings, and Support stay distinct through sidebar groups, not a second top-level mode control.

chosen boundary
+
+
rclone-jav
+
+ +
+
+
Console

Maintenance

Review the library repeatedly.

+
Settings

Configure

Profiles, rules, overlays, deletion.

+
Support

Troubleshoot

Diagnostics, debug benches, setup.

+
+
+

Settings mode would look quieter

+
+
+
+
+
+

Chosen conceptually, simplified visually: sidebar groups do the separating work.

+
+ +
+

4. No Workflow Wizard

The maintenance order is real, but it should be encoded by the Console tools themselves rather than another page.

rejected surface
+
+
Console Order
+
+

Maintenance stays obvious

The sidebar and focused tools make the flow clear without adding a separate wizard surface.

+
+
1
Refresh cache
Update changed files from configured roots.
+
2
Review skipped names
Spot files that did not produce an ID.
+
3
Review duplicates
KEEP reasons and multipart-risk skips included.
+
4
Check library issues
Rename bracket/no-hyphen oddities if needed.
+
+
Utility
+
+
+

Rejected as a dedicated home page. Useful order, unnecessary extra destination.

+
+ +
+

5. Rejected: Bulk Mode Inside Popup

This would turn the popup into a two-mode mini-app with cramped result review.

rejected
+
+
+
rclone-jav
LIVECACHE
+
SingleBulk
+ +
+
6 IDs checked · 4 match · 2 no match
+
+
BLK-474MATCH
+
PRTD-029NO MATCH
+
FC2-PPV-1841460MATCH
+
+
+
+

Rejected even for 5-20 IDs. The popup gets a doorway into a focused Bulk Check surface, not a permanent second mode.

+
+ +
+

6. Chosen: Bulk Check Quick Window

Typical batches are expected to be about 5-20 IDs, so the popup opens a compact focused window while the Console owns the full tool.

chosen bulk path
+
+
+
+
rclone-jav
+ +
MATCH · 1 hit
+
+
+
+

Bulk Check

Quick batch window

+ +
+
3 IDs · 2 match · 1 no match
+
+
BLK-474MATCH
+
PRTD-027NO MATCH
+
+
+
+
+

Console Owner

+ +
+

Full-width rows, richer review, ranges, import/export, future filters.

+
+
+
+

Chosen: popup opens a compact Bulk Check window for short batches. The Console remains the full batch-review surface.

+
+ +
+

7. Debug Split + Repo Preview

Debug history and standalone tests move out of daily workflow. Sim Dupe leaves the extension UI entirely.

chosen support split
+
+
Support / Debug Tools
+
+
+

Debug Tools

+
Search ActivitySearch TroubleshootingID ExtractionPage ExtractionDiagnostics
+
+
+

Repo Preview Harness

Popup state samples live in a repo HTML file such as samples/popup-states.html, not as a hidden extension page.

+
+
+
MATCH · sample popup state
+
BLK-474 - ClearJAV.mp44.94 GiB
BLK-474 [1080p].mp44.90 GiB
+
+
+
+
+

Chosen: standalone support/debug tools remain available; Sim Dupe is removed from extension UI.

+
+ +
+

8. Inline Rule Feedback

Rule editors keep local feedback. Only standalone troubleshooting benches move to Debug Tools.

chosen bench split
+
+
Settings / Matching Rulesfeedback stays nearby
+
+
Custom Part DetectorID NormalizerSite Extraction
+
+
+

Rule

+ +
+
+
+

Inline Feedback

+
+
KV-118_PART1.mp4part 1
+
KV-118_PART2.mp4part 2
+
Covered by built-in?shown
+
+
+
+

Standalone Search Troubleshooting, page extraction testing, and search history can still move into Support. Editing rules should not require leaving the editor to see feedback.

+
+
+

Chosen: contextual inline tests stay. General troubleshooting tools move to Support.

+
+
+
+ + diff --git a/rc-jav.py b/rc-jav.py new file mode 100644 index 0000000..37fc844 --- /dev/null +++ b/rc-jav.py @@ -0,0 +1,2230 @@ +#!/usr/bin/env python3 +"""Scan rclone remotes for duplicate JAV files grouped by ID.""" +from __future__ import annotations + +import argparse +import csv +import fnmatch +import json +import os +import re +import subprocess +import sys +import threading +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass, asdict +from datetime import datetime +from pathlib import Path +from typing import Iterable + +from rich.console import Console +from rich.panel import Panel +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) +from rich.table import Table +from rich.text import Text + +PRIMARY_ID_RE = re.compile(r"^([A-Za-z]+)-(\d+)") +FALLBACK_ID_RE = re.compile(r"^([A-Za-z0-9]+)-(\d+)") +COMPOUND_ID_RE = re.compile(r"^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)+)-(\d+)") + +# Part-suffix patterns: anchored at end of stem (after stripping extension). +# Each pattern's group(1) is the part number. +RANGE_RE = re.compile(r"\[(\d+)-(\d+)\]") + +# Non-anchored XofY probe used in detect_part() to resolve the priority conflict +# between a trailing (N) copy-marker suffix and an embedded XofY part indicator. +# Example: "ENKI-031 [1080p].2of2 (1)" — the (1) is a filesystem collision suffix +# (rclone, Windows copy), not a part number; the 2of2 is the real part indicator. +# This pattern intentionally has no end-anchor so it matches anywhere in the stem. +_XOFY_PRIORITY_RE = re.compile(r"[._ -](\d+)\s*of\s*\d+", re.IGNORECASE) + +BUILTIN_PART_RES = [ + re.compile(r"[-_ ](?:pt|part|cd|disc)[-_ ]?(\d+)$", re.IGNORECASE), + re.compile(r"\s*\((\d+)(?:\s*of\s*\d+)?\)$", re.IGNORECASE), + # Exported multipart filenames often end in `.1of2` / `-2 of 4`. + re.compile(r"[._ -](\d+)\s*of\s*\d+$", re.IGNORECASE), + # Bare numeric suffixes (`_N`, ` N`) are only treated as part numbers when + # the number is 1-2 digits. Wider patterns falsely matched resolution tags + # (`_2160`, `_4K2160`) and dates/years (`SSIS-001 2023.mp4` -> `#part2023`), + # corrupting cache keys. + # Staged detection also retries after resolution/actress cleanup, so end + # anchors can match both raw suffixes and metadata-blocked suffixes safely. + re.compile(r"_(\d{1,2})$"), + # Hyphen short-part suffix after the ID, e.g. OFJE-195-1 [480p].mp4. + # Limit to 1-2 digits so the base ID's usual 3+ digit numeric component + # does not make every canonical `ABC-123` filename look multipart. + re.compile(r"-(\d{1,2})$"), + # Lettered parts: separator (hyphen or underscore) followed by A-D. + # Uppercase only — lowercase letters are variant designators (e.g. IBW-902z) + # and are preserved as part of the base ID, not treated as part numbers. + re.compile(r"[-_]([A-D])$"), + # Bare uppercase letter directly after the ID digits with no separator, + # e.g. BAK-052A, BAK-052B. Lookbehind ensures a digit precedes. + re.compile(r"(?<=\d)([A-D])$"), + re.compile(r"\s+(\d{1,2})$"), +] +PART_RES = list(BUILTIN_PART_RES) + + +def configure_part_patterns(patterns: Iterable[str]) -> list[str]: + """Extend part suffix detection with user regexes whose first group is part number.""" + global PART_RES + PART_RES = list(BUILTIN_PART_RES) + errors: list[str] = [] + for pattern in patterns: + pattern = str(pattern or "").strip() + if not pattern: + continue + try: + compiled = re.compile(pattern, re.IGNORECASE) + except re.error as e: + errors.append(f"{pattern!r}: {e}") + continue + if compiled.groups < 1: + errors.append(f"{pattern!r}: needs a capture group for the part number") + continue + PART_RES.append(compiled) + return errors + + +def detect_part(stem: str) -> str | None: + """Return part number as string if stem ends with a part suffix, else None. + + XofY (e.g. .2of2) anywhere in the stem takes unconditional priority over a + trailing (N) suffix. A file named 'ENKI-031 [1080p].2of2 (1).mp4' is part 2; + the trailing (1) is a filesystem copy-collision marker (rclone / Windows), + not a part number. Without this pre-check the ordered PART_RES list would + match (1) first and misclassify the file as part 1. + """ + m = _XOFY_PRIORITY_RE.search(stem) + if m: + return m.group(1) + for r in PART_RES: + m = r.search(stem) + if m: + return m.group(1) + return None + + +def part_key(part: str) -> str: + token = str(part or "").strip() + if token.isdigit(): + return str(int(token)) + if len(token) == 1 and token.isalpha(): + return str(ord(token.upper()) - ord("A") + 1) + return token.upper() + + +@dataclass +class FileEntry: + source: str # "Source" (priority) or "Target" + remote: str # the rclone remote:path root supplied + path: str # relative path within remote + size: int + mod_time: str + jav_id: str # normalized, e.g. "SSIS-1" + + @property + def full_path(self) -> str: + sep = "" if self.remote.endswith("/") or not self.path else "/" + return f"{self.remote}{sep}{self.path}" + + + +def human_size(n: int) -> str: + nf = float(max(0, n)) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if nf < 1024: + return f"{int(nf)} B" if unit == "B" else f"{nf:.2f} {unit}" + nf /= 1024 + return f"{nf:.2f} PiB" + + +# Matches a trailing lowercase letter variant designator, e.g. the 'z' in IBW-902z. +_VARIANT_SUFFIX_RE = re.compile(r"^(.+?)([a-z])$") + +# Strips `[resolution]` and ` - Actress Name` from a stem so that part-suffix +# patterns anchored at `$` fire correctly. +# Canonical naming: {ID}[-{part}][ - {actress}][ [{resolution}]] +_RESOLUTION_TAG_RE = re.compile(r"\s*\[[^\]]*\]\s*$") + +# Bracket-wrapped ID: [REAL-779] or [HODV-21076] Saki Hatsumi [1080p] +_BRACKET_ID_RE = re.compile(r"^\[([^\]]+)\]") +_RES_LABEL_RE = re.compile(r"\[(?:2160|1080|720|480)p\]", re.IGNORECASE) +_VIDEO_EXTS = { + ".avi", ".flv", ".m2ts", ".m4v", ".mkv", ".mov", ".mp4", ".mpeg", + ".mpg", ".ts", ".webm", ".wmv", +} +_LOWEST_KEEP_PRIORITY_EXTS = {".ts"} + +# No-hyphen ID fallback: MVSD312 → MVSD-312 (letters-only prefix + digits, no hyphen) +_NOHYPHEN_ID_RE = re.compile(r"^([A-Za-z]{2,8})(\d{3,6})") + + +def _clean_stem_for_parts(stem: str) -> str: + """Return stem with trailing [tag] and ' - Actress' stripped. + Resolution is always the last bracketed token; actress follows ' - '.""" + s = _RESOLUTION_TAG_RE.sub("", stem).strip() + if " - " in s: + s = s[:s.index(" - ")].strip() + return s + + +def _part_detection_stems(stem: str) -> list[str]: + """Return stem stages for part detection from least to most cleaned.""" + resolution_clean = _RESOLUTION_TAG_RE.sub("", stem).strip() + actress_clean = _clean_stem_for_parts(stem) + out: list[str] = [] + for candidate in (stem, resolution_clean, actress_clean): + if candidate and candidate not in out: + out.append(candidate) + return out + + +def detect_part_from_stem(stem: str) -> str | None: + """Try part suffix rules before and after metadata cleanup.""" + for candidate in _part_detection_stems(stem): + part = detect_part(candidate) + if part: + return part + return None + + +def extract_id(name: str) -> str | None: + stem = Path(name).stem + + # Strip bracket wrapper: [REAL-779] → REAL-779, [SCOP-297] [1080p] → SCOP-297 + effective_stem = stem + if stem.startswith("["): + bm = _BRACKET_ID_RE.match(stem) + if bm: + effective_stem = bm.group(1).strip() + + m = PRIMARY_ID_RE.match(effective_stem) + if not m: + m = COMPOUND_ID_RE.match(effective_stem) + if not m: + m = FALLBACK_ID_RE.match(effective_stem) + if not m: + # No-hyphen fallback: MVSD312 → MVSD-312 + m = _NOHYPHEN_ID_RE.match(effective_stem) + if not m: + return None + + num = int(m.group(2)) + width = max(3, len(m.group(2))) + prefix = m.group(1).upper() + if prefix == "FC2": + prefix = "FC2-PPV" + + # Check the character immediately after the matched digits. + # Lowercase → variant designator (e.g. IBW-902z): fold into the base ID. + # Uppercase A-D → part letter: handled below by detect_part. + # Anything else (space, '[', end-of-string) → no variant. + after = effective_stem[m.end():m.end() + 1] + variant = after if after.islower() else "" + + base = f"{prefix}-{num:0{width}d}{variant}" + + # Use original stem (not effective_stem) so bracket-wrapped filenames like + # [REAL-779-1].mp4 still get part detection applied to the full stem. + # Run before and after metadata cleanup: raw suffixes such as + # "KV-118 - Actress_PART1" must survive, while trailing [1080p] tags still + # need cleanup before end-anchored detectors can match. + part = detect_part_from_stem(stem) + return f"{base}#part{part_key(part)}" if part else base + + +def normalize_id(raw: str) -> str | None: + return extract_id(raw + ".x") # add dummy ext so stem keeps the ID intact + + +def describe_id_match(display_query: str, matched_query: str, matched_id: str, + expansion_count: int) -> dict[str, str]: + """Explain the matcher path used for one ID hit in JSON output.""" + if "*" in matched_query or "?" in matched_query: + kind, label, confidence = "wildcard", "Wildcard ID", "broad" + elif expansion_count > 1: + kind, label, confidence = "range", "Range member", "expanded" + elif "#part" in matched_query: + kind, label, confidence = "exact_part", "Exact part ID", "high" + elif matched_id.startswith(matched_query + "#part"): + kind, label, confidence = "part", "Base ID + part", "related" + elif display_query.upper() != matched_query.upper(): + kind, label, confidence = "normalized", "Normalized ID", "normalized" + else: + kind, label, confidence = "exact", "Exact ID", "high" + return { + "match_kind": kind, + "match_reason": label, + "match_confidence": confidence, + "matched_query": matched_query, + "matched_id": matched_id, + } + + +def expand_range(raw: str) -> list[str] | None: + """Expand a bracket range like 'IPZZ-[820-860]' into individual ID strings. + Returns None if no range marker present.""" + m = RANGE_RE.search(raw) + if not m: + return None + a, b = int(m.group(1)), int(m.group(2)) + lo, hi = (a, b) if a <= b else (b, a) + width = max(len(m.group(1)), len(m.group(2))) # preserve zero-padding + return [raw[:m.start()] + f"{n:0{width}d}" + raw[m.end():] for n in range(lo, hi + 1)] + + +RCLONE_BIN = "rclone" +BASIC = False # set by --basic +USE_ANSI = True # disabled by --no-color + +# Pre-rich ANSI codes (used in --basic mode for color). +ANSI_RESET = "\033[0m" +ANSI_GREEN = "\033[32m" +ANSI_RED = "\033[31m" +ANSI_YELLOW = "\033[33m" +ANSI_CYAN = "\033[36m" +ANSI_DIM = "\033[2m" +ANSI_BOLD = "\033[1m" + + +def ansi(s: str, code: str) -> str: + return f"{code}{s}{ANSI_RESET}" if USE_ANSI else s +console = Console() # replaced in main() if --no-color + + +_RICH_TAG_RE = re.compile(r"\[/?[^\]]*\]") + + +def strip_markup(s: str) -> str: + return _RICH_TAG_RE.sub("", s) + + +class BasicProgress: + """Minimal stand-in for rich.Progress used when --basic is set.""" + def __init__(self): + self._tasks: dict[int, dict] = {} + self._next = 0 + self._last_print: dict[int, int] = {} + + def __enter__(self): + return self + + def __exit__(self, *exc): + for tid, t in self._tasks.items(): + sys.stderr.write(f"{ansi('[done]', ANSI_GREEN)} {t['desc']} {t['done']}/{t['total']}\n") + return False + + def add_task(self, description: str, total: int = 1) -> int: + tid = self._next + self._next += 1 + desc = strip_markup(description) + self._tasks[tid] = {"desc": desc, "total": total, "done": 0} + self._last_print[tid] = 0 + sys.stderr.write(f"{ansi('[start]', ANSI_CYAN)} {desc}\n") + return tid + + def update(self, tid, total=None, description=None, **_): + t = self._tasks[tid] + if total is not None: + t["total"] = total + if description is not None: + t["desc"] = strip_markup(description) + + def advance(self, tid, n: int = 1): + t = self._tasks[tid] + t["done"] += n + # In-place refresh every 5 files (or every file if total small). + step = 5 if t["total"] > 50 else 1 + if t["done"] - self._last_print[tid] >= step or t["done"] == t["total"]: + counter = ansi(f"{t['done']}/{t['total']}", ANSI_CYAN) + line = f" {counter} {ansi(t['desc'], ANSI_DIM)}" + if sys.stderr.isatty(): + sys.stderr.write(f"\r\033[K{line}") + if t["done"] == t["total"]: + sys.stderr.write("\n") + sys.stderr.flush() + elif t["done"] == t["total"]: + # Non-TTY: only print final line, skip intermediate noise. + sys.stderr.write(line + "\n") + self._last_print[tid] = t["done"] + +# Default remotes used when --search is invoked without explicit --source/--target. +DEFAULT_SOURCE = ["cq:personal-files/ClearJAV"] +DEFAULT_TARGET = ["cq:personal-files/JAV/TMP"] + +# Default WinCatalog export folder (or specific files). Folder entries auto-discover *.csv / *.xml. +DEFAULT_CATALOG: list[str] = [str(Path(__file__).resolve().parent / "wincatalog")] + +# CSV column synonyms (lowercased) — first matching one wins. +CATALOG_COL_NAME = ("name", "file name", "filename", "title") +CATALOG_COL_PATH = ("path", "full path", "location", "folder") +CATALOG_COL_SIZE = ("size", "file size", "bytes", "size (bytes)") +CATALOG_COL_DISC = ("disc", "disc name", "disc label", "volume", "source", "catalog", "media") + +CACHE_PATH = Path(__file__).resolve().parent / "cache.json" +CACHE_VERSION = 3 # bumped: extract_id handles bracket-wrapped IDs + no-hyphen fallback +CACHE_STALE_HOURS = 24 + +DEFAULT_KEEP_RANKING: dict = { + "priority_folders": ["ClearJAV"], + "size_tolerance_mib": 0, + "format_preference": ["mkv", "mp4", "wmv", "avi"], + "tiebreak_res_tag": True, + "tiebreak_longer_name": True, +} +# Module-level ranking config; set from config.json in main() so all call sites pick it up. +_KEEP_RANKING: dict = {} + +CONFIG_PATH = Path(__file__).resolve().parent / "config.json" + +# Written by the native-messaging host when the user clicks Cancel in the +# extension popup. walk_remote checks for it every CANCEL_CHECK_INTERVAL files +# and exits cleanly if found. +CANCEL_FLAG = Path(__file__).resolve().parent / "scan-cancel.flag" +CANCEL_CHECK_INTERVAL = 100 # check / emit progress every N files + + +def load_config() -> dict: + if not CONFIG_PATH.exists(): + return {} + try: + data = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return {} + return data + except (json.JSONDecodeError, OSError): + return {} + + +def save_config(cfg: dict) -> None: + tmp = CONFIG_PATH.with_suffix(CONFIG_PATH.suffix + ".tmp") + tmp.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + os.replace(tmp, CONFIG_PATH) + + +def load_cache() -> dict: + if not CACHE_PATH.exists(): + return {"version": CACHE_VERSION, "remotes": {}} + try: + data = json.loads(CACHE_PATH.read_text(encoding="utf-8")) + if ( + not isinstance(data, dict) + or data.get("version") != CACHE_VERSION + or not isinstance(data.get("remotes"), dict) + ): + if isinstance(data, dict) and "version" in data and data["version"] != CACHE_VERSION: + sys.stderr.write( + f"[warn] cache version mismatch (got {data['version']}, " + f"expected {CACHE_VERSION}); forcing full rescan.\n" + ) + return {"version": CACHE_VERSION, "remotes": {}} + return data + except (json.JSONDecodeError, OSError): + return {"version": CACHE_VERSION, "remotes": {}} + + +def save_cache(cache: dict) -> None: + # Write to a sibling tmp file then atomically replace, so a killed mid-write + # (Ctrl-C, power loss, concurrent --scan) can't leave a half-written + # cache.json — load_cache would otherwise see invalid JSON and fall back to + # an empty cache, forcing a full re-scan. + tmp = CACHE_PATH.with_suffix(CACHE_PATH.suffix + ".tmp") + tmp.write_text(json.dumps(cache, indent=2), encoding="utf-8") + try: + os.replace(tmp, CACHE_PATH) + except PermissionError: + # Windows: destination may be briefly locked by antivirus or a concurrent reader. + time.sleep(0.5) + os.replace(tmp, CACHE_PATH) + + +def cache_age_hours(scanned_at: str) -> float | None: + try: + dt = datetime.fromisoformat(scanned_at.replace("Z", "+00:00")) + except ValueError: + return None + now = datetime.now(dt.tzinfo) if dt.tzinfo else datetime.now() + return (now - dt).total_seconds() / 3600.0 + + +def fmt_age(hours: float) -> str: + if hours < 1: + return f"{int(hours * 60)}m" + if hours < 24: + return f"{hours:.1f}h" + return f"{hours / 24:.1f}d" + + +# ---------- WinCatalog ingest ---------- + +def _pick_col(headers_lower: list[str], synonyms: tuple[str, ...]) -> str | None: + for s in synonyms: + if s in headers_lower: + return s + return None + + +def normalize_catalog_path(path: str) -> str: + """Keep catalog paths display-compatible with rclone-style path consumers.""" + p = (path or "").replace("\\", "/") + if p.startswith("//"): + return "//" + re.sub(r"/+", "/", p[2:]) + return re.sub(r"/+", "/", p) + + +def load_catalog_csv(path: Path, skipped: list[tuple[str, str]]) -> list[FileEntry]: + """Load a WinCatalog CSV export. Lenient about column names.""" + entries: list[FileEntry] = [] + with path.open("r", encoding="utf-8-sig", newline="") as f: + # Sniff delimiter + sample = f.read(4096) + f.seek(0) + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") + except csv.Error: + dialect = csv.excel + reader = csv.DictReader(f, dialect=dialect) + if not reader.fieldnames: + return entries + headers: dict[str, str] = {} + for h in reader.fieldnames: + hl = h.lower() + if hl not in headers: + headers[hl] = h + col_name = _pick_col(list(headers), CATALOG_COL_NAME) + col_path = _pick_col(list(headers), CATALOG_COL_PATH) + col_size = _pick_col(list(headers), CATALOG_COL_SIZE) + col_disc = _pick_col(list(headers), CATALOG_COL_DISC) + if not col_name and not col_path: + console.print(f"[yellow]WARN: catalog CSV {path} has no Name/Path columns; skipping.[/]") + return entries + for row in reader: + name = (row.get(headers[col_name]) if col_name else "") or "" + full_path = (row.get(headers[col_path]) if col_path else "") or "" + if not name and full_path: + name = Path(full_path).name + full_path = normalize_catalog_path(full_path) + if not name: + continue + jav_id = extract_id(name) + if not jav_id: + skipped.append((f"catalog:{path.name}", full_path or name)) + continue + try: + size = int(row.get(headers[col_size], 0)) if col_size else 0 + except (ValueError, TypeError): + size = 0 + disc = (row.get(headers[col_disc]) if col_disc else "") or "" + # Encode disc label into "remote" so it surfaces in output. + remote_label = f"catalog:{disc}" if disc else f"catalog:{path.name}" + entries.append(FileEntry( + source="Catalog", remote=remote_label, + path=full_path or name, size=size, mod_time="", + jav_id=jav_id, + )) + return entries + + +def _strip_xml_ns(tag: str) -> str: + """Remove Clark-notation namespace {uri}local → local.""" + return tag.split("}")[-1] if "}" in tag else tag + + +def load_catalog_xml(path: Path, skipped: list[tuple[str, str]]) -> list[FileEntry]: + """Load a WinCatalog XML export. Walks for any element with file-like attrs.""" + entries: list[FileEntry] = [] + tree = ET.parse(str(path)) + root = tree.getroot() + + def walk(node, disc_label: str, parent_path: str, _depth: int = 0): + if _depth > 500: + return + tag = _strip_xml_ns(node.tag).lower() + # Heuristics: disc/catalog/source containers reset disc_label + if tag in ("disc", "catalog", "source", "volume", "media"): + disc_label = node.get("name") or node.get("Name") or disc_label + # File-like nodes + if tag in ("file", "f"): + name = node.get("name") or node.get("Name") or node.findtext("Name") or "" + size_raw = node.get("size") or node.get("Size") or node.findtext("Size") or "0" + try: + size = int(size_raw) + except ValueError: + size = 0 + full_path = normalize_catalog_path(f"{parent_path}/{name}" if parent_path else name) + jav_id = extract_id(name) + if jav_id: + entries.append(FileEntry( + source="Catalog", + remote=f"catalog:{disc_label}" if disc_label else f"catalog:{path.name}", + path=full_path, size=size, mod_time="", jav_id=jav_id, + )) + else: + skipped.append((f"catalog:{disc_label or path.name}", full_path)) + return + # Folder-like: extend parent_path + if tag in ("folder", "dir", "directory"): + folder_name = node.get("name") or node.get("Name") or "" + parent_path = normalize_catalog_path(f"{parent_path}/{folder_name}" if parent_path else folder_name) + for child in node: + walk(child, disc_label, parent_path, _depth + 1) + + walk(root, "", "") + return entries + + +def _expand_catalog_paths(paths: list[str]) -> list[Path]: + """Expand any directories to their *.csv / *.xml children. Files passed through.""" + out: list[Path] = [] + for p in paths: + cp = Path(p) + if cp.is_dir(): + for child in sorted(cp.iterdir()): + if child.suffix.lower() in (".csv", ".xml") and child.is_file(): + out.append(child) + elif cp.exists(): + out.append(cp) + # silently skip missing default dir; warn for everything else + elif Path(p).resolve() not in {Path(d).resolve() for d in DEFAULT_CATALOG}: + console.print(f"[yellow]WARN: catalog path not found: {p}[/]") + return out + + +def load_catalogs(paths: list[str], skipped: list[tuple[str, str]]) -> list[FileEntry]: + out: list[FileEntry] = [] + for cp in _expand_catalog_paths(paths): + ext = cp.suffix.lower() + if ext == ".csv": + out.extend(load_catalog_csv(cp, skipped)) + elif ext == ".xml": + out.extend(load_catalog_xml(cp, skipped)) + else: + console.print(f"[yellow]WARN: unknown catalog format '{ext}' for {cp}; skipping.[/]") + return out + + +# ---------- quick search (no cache) ---------- + +def quick_search_remote(remote: str, source_label: str, + patterns: list[str], + skipped: list[tuple[str, str]]) -> list[FileEntry]: + """Run `rclone lsjson --include ` once per pattern. Bypass cache.""" + out: list[FileEntry] = [] + seen: set[tuple[str, str]] = set() + for pat in patterns: + cmd = [RCLONE_BIN, "lsjson", remote, "--files-only", "-R", "--include", pat] + proc = subprocess.run(cmd, capture_output=True, text=True, + encoding="utf-8", errors="replace") + if proc.returncode != 0: + console.print(f"[red]rclone lsjson --include failed for {remote}:[/]\n{proc.stderr}") + sys.exit(proc.returncode) + for item in json.loads(proc.stdout or "[]"): + if item.get("IsDir"): + continue + path = item["Path"] + key = (remote, path) + if key in seen: + continue + seen.add(key) + jav_id = extract_id(Path(path).name) + if not jav_id: + skipped.append((remote, path)) + continue + out.append(FileEntry( + source=source_label, remote=remote, path=path, + size=int(item.get("Size", 0)), + mod_time=item.get("ModTime", ""), jav_id=jav_id, + )) + return out + + +def choose_search_mode(raw_queries: list[str], force_quick: bool, force_cache: bool) -> tuple[str, str]: + """Decide quick vs cached. Returns (mode, reason).""" + if force_quick and force_cache: + return ("cached", "both --quick and --cache passed; preferring --cache (safer)") + if force_quick: + return ("quick", "forced via --quick") + if force_cache: + return ("cached", "forced via --cache") + if len(raw_queries) > 1: + return ("cached", f"multi-query ({len(raw_queries)} IDs) — cache batches them for free") + if not raw_queries: + return ("cached", "no queries") + q = raw_queries[0] + if RANGE_RE.search(q): + return ("cached", "range [N-M] — too many rclone calls otherwise") + if "*" in q or "?" in q: + return ("cached", "wildcard — cache match semantics are more reliable") + return ("quick", "single exact ID — live lookup is fastest") + + +def _escape_rclone_glob(s: str) -> str: + """Escape rclone filter meta-chars so a literal token isn't interpreted as a + glob. rclone's filter syntax treats `*`, `?`, `[`, `{` specially; brackets + open a char-class that fails silently if the token contains `[` or `]`.""" + out = [] + for ch in s: + if ch in r"*?[]{}\\": + out.append("\\" + ch) + else: + out.append(ch) + return "".join(out) + + +def name_to_include_patterns(tokens: list[str]) -> list[str]: + """Build rclone --include globs for each name token (case-insensitive substring).""" + pats: list[str] = [] + for t in tokens: + if "*" in t or "?" in t: + # Caller-supplied wildcard — assume they meant it. + pats.append(t) + else: + # Literal substring search: escape glob meta inside the token so + # `--name "[BD]"` searches for the literal "[BD]" not a char class. + pats.append(f"*{_escape_rclone_glob(t)}*") + return pats + + +def name_match(stem: str, tokens: list[str]) -> bool: + """Case-insensitive: True if ANY token matches stem (substring or fnmatch glob).""" + low = stem.lower() + for t in tokens: + tl = t.lower() + if "*" in tl or "?" in tl: + if fnmatch.fnmatchcase(low, tl): + return True + elif tl in low: + return True + return False + + +def query_to_include_patterns(raw: str) -> list[str]: + """Turn a search query into one or more rclone --include globs. + Ranges expand to individual IDs; wildcards and exact IDs map to single glob.""" + if RANGE_RE.search(raw): + expanded = expand_range(raw) or [] + out: list[str] = [] + for e in expanded: + out.extend(query_to_include_patterns(e)) + return out + if "*" in raw or "?" in raw: + return [f"{raw}*"] + norm = normalize_id(raw) + if not norm: + return [f"{raw}*"] + prefix, _, digits = norm.rpartition("-") + if not digits.isdigit(): + return [f"{norm}*"] + n = int(digits) + width = max(3, len(str(n))) + return [f"{prefix}-{n:0{width}d}*"] + + +# ---------- rclone wrappers ---------- + +def remote_file_count(remote: str) -> int: + """Fast total file count via `rclone size --json`.""" + cmd = [RCLONE_BIN, "size", "--json", remote] + proc = subprocess.run(cmd, capture_output=True, text=True, + encoding="utf-8", errors="replace") + if proc.returncode != 0: + console.print(f"[red]rclone size failed for {remote}:[/]\n{proc.stderr}") + sys.exit(proc.returncode) + try: + return int(json.loads(proc.stdout).get("count", 0)) + except (json.JSONDecodeError, ValueError): + return 0 + + +DURATION_RE = re.compile(r"^\s*(\d+)\s*([smhd])\s*$", re.IGNORECASE) + + +def parse_duration(s: str) -> str | None: + """Validate a duration suffix (`30m`, `24h`, `7d`, `90s`). Returns the + normalized form rclone accepts, or None if invalid. We don't compute a + timedelta — we pass the suffix straight to rclone --max-age.""" + if not s: + return None + m = DURATION_RE.match(s) + if not m: + return None + return f"{m.group(1)}{m.group(2).lower()}" + + +def walk_remote(remote: str, source_label: str, + skipped: list[tuple[str, str]], + progress: Progress, task_id, + max_age: str | None = None, + _total_override: int | None = None) -> tuple[list[FileEntry], list[str]]: + """Stream files from rclone lsf, ticking progress per file. + If max_age is set, pass --max-age to rclone so only recently-modified files + are returned (incremental scan). + _total_override: skip the internal remote_file_count probe (caller already did it).""" + if max_age: + # Can't pre-count for an age-filtered walk — skip the size probe and + # let progress run on a synthetic total. + total = 0 + progress.update(task_id, total=1, + description=f"[cyan]{source_label}[/] {remote} (since {max_age})") + else: + if _total_override is not None: + total = _total_override + else: + total = remote_file_count(remote) + if BASIC: + # Caller already emitted SCAN_REMOTE_START (without total) — now we know it. + sys.stderr.write("SCAN_REMOTE_COUNTED " + json.dumps({ + "remote": remote, "total": total, + }) + "\n") + sys.stderr.flush() + progress.update(task_id, total=max(total, 1), + description=f"[cyan]{source_label}[/] {remote}") + cmd = [RCLONE_BIN, "lsf", "--files-only", "-R", + "--format", "pst", "--separator", "\t"] + if max_age: + cmd += ["--max-age", max_age] + cmd.append(remote) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding="utf-8", errors="replace") + entries: list[FileEntry] = [] + local_skipped: list[str] = [] + if proc.stdout is None: + raise RuntimeError("rclone stdout pipe unexpectedly None") + _stderr_chunks: list[str] = [] + _stderr_thread = threading.Thread( + target=lambda: _stderr_chunks.append(proc.stderr.read() if proc.stderr else ""), + daemon=True, + ) + _stderr_thread.start() + _cancelled = False + try: + for line in proc.stdout: + line = line.rstrip("\n").rstrip("\r") + if not line: + continue + parts = line.split("\t") + if len(parts) < 2: + continue + rel = parts[0] + try: + size = int(parts[1]) + except ValueError: + size = 0 + mod_time = parts[2] if len(parts) >= 3 else "" + jav_id = extract_id(Path(rel).name) + if not jav_id: + local_skipped.append(rel) + skipped.append((remote, rel)) + else: + entries.append(FileEntry( + source=source_label, remote=remote, path=rel, + size=size, mod_time=mod_time, jav_id=jav_id, + )) + progress.advance(task_id) + # Every CANCEL_CHECK_INTERVAL files: check cancel flag and emit progress. + n = len(entries) + len(local_skipped) + if BASIC and n > 0 and n % CANCEL_CHECK_INTERVAL == 0: + if CANCEL_FLAG.exists(): + try: + CANCEL_FLAG.unlink(missing_ok=True) + except OSError: + pass + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + _cancelled = True + break + sys.stderr.write("SCAN_FILE_PROGRESS " + json.dumps({ + "remote": remote, "label": source_label, + "files": len(entries), "skipped": len(local_skipped), + "total": total, + }) + "\n") + sys.stderr.flush() + except KeyboardInterrupt: + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + raise + if _cancelled: + sys.stderr.write("SCAN_CANCELLED\n") + sys.stderr.flush() + sys.exit(0) + proc.wait() + _stderr_thread.join() + if proc.returncode != 0: + err = _stderr_chunks[0] if _stderr_chunks else "" + console.print(f"[red]rclone lsf failed for {remote}:[/]\n{err}") + sys.exit(proc.returncode) + return entries, local_skipped + + +def make_progress(): + if BASIC: + return BasicProgress() + return Progress( + SpinnerColumn(), + TextColumn("{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TimeElapsedColumn(), + TextColumn("eta"), + TimeRemainingColumn(), + console=console, + transient=False, + ) + + +# ---------- collectors ---------- + +def collect_with_progress(remotes_by_label: list[tuple[str, str]], + skipped: list[tuple[str, str]] + ) -> list[FileEntry]: + """Dupe-mode collect — every remote freshly walked with progress.""" + out: list[FileEntry] = [] + if not remotes_by_label: + return out + with make_progress() as progress: + tasks = {(label, r): progress.add_task(f"{label} {r}", total=1) + for label, r in remotes_by_label} + for (label, r), tid in tasks.items(): + entries, _ = walk_remote(r, label, skipped, progress, tid) + out.extend(entries) + return out + + +def cached_collect(remotes: list[str], source_label: str, + skipped: list[tuple[str, str]], + cache: dict, use_cache: bool, force_update: bool, + cache_meta: dict[str, dict], + scan_since: str | None = None) -> list[FileEntry]: + """Search-mode collect with cache. Always recursive. + scan_since: rclone duration string (`24h`, `7d`). When set during a forced + update, only files modified within the window are walked and merged on top + of the existing cache entry; files older than the window keep their cached + record. If there's no prior cache entry for a remote, falls through to a + full scan.""" + out: list[FileEntry] = [] + to_scan: list[str] = [] + to_incremental: list[tuple[str, dict]] = [] # (remote, existing_entry) + for r in remotes: + if scan_since and force_update and use_cache: + existing = cache["remotes"].get(r) + if existing: + to_incremental.append((r, existing)) + continue + # No prior cache for this remote -> can't be incremental, fall back. + entry = cache["remotes"].get(r) if use_cache and not force_update else None + if entry: + age = cache_age_hours(entry["scanned_at"]) + age_str = fmt_age(age) if age is not None else "?" + stale = age is not None and age > CACHE_STALE_HOURS + cache_meta[r] = {"cached": True, "age": age_str, "stale": stale, + "file_count": len(entry["files"])} + for f in entry["files"]: + out.append(FileEntry(source=source_label, remote=r, path=f["path"], + size=f["size"], mod_time=f.get("mod_time", ""), + jav_id=f["jav_id"])) + for s in entry.get("skipped", []): + skipped.append((r, s)) + else: + to_scan.append(r) + + if to_scan: + with make_progress() as progress: + tids = {r: progress.add_task(f"{source_label} {r}", total=1) for r in to_scan} + for r_idx, r in enumerate(to_scan): + _total: int | None = None + if BASIC: + # Emit SCAN_REMOTE_START immediately so the UI shows the remote name. + # Then probe the file count; once known, emit SCAN_REMOTE_COUNTED so + # the UI can show "N / total" without waiting for the first 100 files. + sys.stderr.write("SCAN_REMOTE_START " + json.dumps({ + "remote": r, "label": source_label, + "index": r_idx + 1, "of": len(to_scan), + "total": None, + }) + "\n") + sys.stderr.flush() + _total = remote_file_count(r) + sys.stderr.write("SCAN_REMOTE_COUNTED " + json.dumps({ + "remote": r, "total": _total, + }) + "\n") + sys.stderr.flush() + fresh, local_skipped = walk_remote(r, source_label, skipped, progress, tids[r], + _total_override=_total) + out.extend(fresh) + cache_meta[r] = {"cached": False, "age": "fresh", "stale": False, + "file_count": len(fresh)} + if use_cache: + cache["remotes"][r] = { + "scanned_at": datetime.now().astimezone().isoformat(), + "recursive": True, + "files": [{"path": e.path, "size": e.size, "mod_time": e.mod_time, + "jav_id": e.jav_id} for e in fresh], + "skipped": local_skipped, + } + if BASIC: + sys.stderr.write("SCAN_PROGRESS " + json.dumps({ + "remote": r, "label": source_label, + "files": len(fresh), "files_total": len(out), + }) + "\n") + sys.stderr.flush() + + if to_incremental: + with make_progress() as progress: + tids = {r: progress.add_task(f"{source_label} {r} (since {scan_since})", total=1) + for r, _ in to_incremental} + for r_idx, (r, existing) in enumerate(to_incremental): + if BASIC: + sys.stderr.write("SCAN_REMOTE_START " + json.dumps({ + "remote": r, "label": source_label, + "index": r_idx + 1, "of": len(to_incremental), + "total": None, "incremental": True, + }) + "\n") + sys.stderr.flush() + fresh, local_skipped = walk_remote( + r, source_label, skipped, progress, tids[r], max_age=scan_since, + ) + # Merge: replace entries at paths we just walked, keep all others. + new_paths = {e.path for e in fresh} + old_files = [f for f in existing.get("files", []) + if f["path"] not in new_paths] + merged_files = old_files + [ + {"path": e.path, "size": e.size, "mod_time": e.mod_time, + "jav_id": e.jav_id} for e in fresh + ] + # Merge skipped lists (de-dupe). + old_skipped = set(existing.get("skipped", [])) + old_skipped.update(local_skipped) + # Emit FileEntry for everything (old + new) so the caller sees the + # full set, not just deltas. + for f in merged_files: + out.append(FileEntry(source=source_label, remote=r, path=f["path"], + size=f["size"], mod_time=f.get("mod_time", ""), + jav_id=f["jav_id"])) + for s in old_skipped: + skipped.append((r, s)) + cache_meta[r] = { + "cached": False, "age": f"incremental {scan_since}", + "stale": False, "file_count": len(merged_files), + "added_or_updated": len(fresh), + } + if use_cache: + cache["remotes"][r] = { + "scanned_at": datetime.now().astimezone().isoformat(), + "recursive": True, + "files": merged_files, + "skipped": sorted(old_skipped), + } + if BASIC: + sys.stderr.write("SCAN_PROGRESS " + json.dumps({ + "remote": r, "label": source_label, + "files": len(fresh), "files_total": len(out), + "incremental": True, + "file_count": len(merged_files), + }) + "\n") + sys.stderr.flush() + return out + + +# ---------- renderers ---------- + +def render_banner(cache_meta: dict[str, dict], mode: str) -> Panel: + lines: list[Text] = [] + lines.append(Text.from_markup(f"[bold]mode:[/] {mode}")) + if cache_meta: + for r, m in cache_meta.items(): + if m["cached"]: + tag = f"CACHED {m['age']}" + (" STALE" if m["stale"] else "") + style = "yellow" if m["stale"] else "dim" + else: + tag = "FRESH SCAN" + style = "green" + lines.append(Text.from_markup( + f" [white]{r}[/] [{style}]{tag}[/] [dim]({m['file_count']} files)[/]" + )) + body = Text("\n").join(lines) + return Panel(body, title="rc-jav", title_align="left", border_style="blue") + + +def render_search(matches: dict[str, list[FileEntry]], queries: list[str], + cache_meta: dict[str, dict]) -> None: + console.print(render_banner(cache_meta, mode="search")) + for q in queries: + hits = matches.get(q, []) + if not hits: + console.print(f"[bold red][{q}] NOT FOUND[/]") + console.print() + continue + title = f"[bold green][{q}] {len(hits)} hit(s)[/]" + tbl = Table(title=title, title_justify="left", show_lines=False, + border_style="green", expand=True) + tbl.add_column("Source", style="yellow", no_wrap=True) + tbl.add_column("Cache", no_wrap=True) + tbl.add_column("File", style="bold", overflow="fold") + tbl.add_column("Size", justify="right", no_wrap=True) + tbl.add_column("Path", style="dim", overflow="fold") + for e in sorted(hits, key=lambda x: (x.jav_id, x.path.lower())): + meta = cache_meta.get(e.remote, {}) + if meta.get("cached"): + cache_tag = "[yellow][CACHED-STALE][/]" if meta.get("stale") else "[dim][CACHED][/]" + else: + cache_tag = "[green][FRESH][/]" + tbl.add_row( + e.source, cache_tag, Path(e.path).name, + f"{human_size(e.size)}\n[dim]({e.size:,} B)[/]", + e.full_path, + ) + console.print(tbl) + console.print() + + +def render_name_matches(hits: list[FileEntry], tokens: list[str], + cache_meta: dict[str, dict]) -> None: + title = f"[bold green]Name match {tokens} — {len(hits)} hit(s)[/]" + if not hits: + console.print(f"[bold red]Name match {tokens} — NOT FOUND[/]") + return + tbl = Table(title=title, title_justify="left", show_lines=False, + border_style="green", expand=True) + tbl.add_column("Source", style="yellow", no_wrap=True) + tbl.add_column("Cache", no_wrap=True) + tbl.add_column("ID", style="bold cyan", no_wrap=True) + tbl.add_column("File", style="bold", overflow="fold") + tbl.add_column("Size", justify="right", no_wrap=True) + tbl.add_column("Path", style="dim", overflow="fold") + for e in sorted(hits, key=lambda x: (x.jav_id, x.path.lower())): + meta = cache_meta.get(e.remote, {}) + if meta.get("cached"): + cache_tag = "[yellow][CACHED-STALE][/]" if meta.get("stale") else "[dim][CACHED][/]" + else: + cache_tag = "[green][FRESH][/]" + tbl.add_row( + e.source, cache_tag, e.jav_id, Path(e.path).name, + f"{human_size(e.size)}\n[dim]({e.size:,} B)[/]", + e.full_path, + ) + console.print(tbl) + console.print() + + +def render_name_matches_plain(hits: list[FileEntry], tokens: list[str], + cache_meta: dict[str, dict]) -> str: + lines: list[str] = [] + if not hits: + lines.append(ansi(f"Name match {tokens} — NOT FOUND", ANSI_RED)) + return "\n".join(lines) + lines.append(ansi(f"Name match {tokens} — {len(hits)} hit(s)", ANSI_GREEN + ANSI_BOLD)) + for e in sorted(hits, key=lambda x: (x.jav_id, x.path.lower())): + meta = cache_meta.get(e.remote, {}) + if meta.get("cached"): + tag = ansi("[CACHED-STALE]", ANSI_YELLOW) if meta.get("stale") else ansi("[CACHED]", ANSI_DIM) + else: + tag = ansi("[FRESH]", ANSI_GREEN) + src = ansi(e.source, ANSI_YELLOW) + lines.append(f" {src} {tag} {ansi(e.jav_id, ANSI_CYAN)}") + lines.append(ansi(f" file: {Path(e.path).name}", ANSI_BOLD)) + lines.append(f" size: {human_size(e.size)} ({e.size:,} bytes)") + lines.append(ansi(f" path: {e.full_path}", ANSI_DIM)) + return "\n".join(lines) + + +def render_dupes(dupes: dict[str, list[FileEntry]], + skipped: list[tuple[str, str]], + variant_alerts: dict[str, list[FileEntry]] | None = None) -> None: + if not dupes: + console.print(Panel("[bold green]No duplicates found.[/]", + border_style="green")) + else: + console.print(f"[bold]Found {len(dupes)} duplicate ID group(s):[/]") + console.print() + total_reclaim = 0 + for jav_id in sorted(dupes): + entries = dupes[jav_id] + keep = decide_keep(entries) + tbl = Table(title=f"[bold][{jav_id}][/]", title_justify="left", + show_lines=False, border_style="magenta", expand=True) + tbl.add_column("Action", no_wrap=True) + tbl.add_column("Source", style="yellow", no_wrap=True) + tbl.add_column("Size", justify="right", no_wrap=True) + tbl.add_column("Path", overflow="fold") + for e in sorted(entries, key=lambda x: (x.source != "Source", x.source == "Catalog", -x.size)): + if e.source == "Catalog": + action = "[cyan]CATALOG[/]" + elif e is keep: + action = "[green]KEEP[/]" + else: + action = "[red]DELETE?[/]" + total_reclaim += e.size + tbl.add_row(action, e.source, + f"{human_size(e.size)}\n[dim]({e.size:,} B)[/]", + e.full_path) + console.print(tbl) + console.print() + console.print(Panel( + f"[bold]Potential space reclaim if all DELETE? removed: " + f"[red]{human_size(total_reclaim)}[/][/]", + border_style="red")) + if skipped: + console.print() + tbl = Table(title=f"[dim]Skipped {len(skipped)} file(s) with no parseable ID[/]", + title_justify="left", show_lines=False, border_style="dim", expand=True) + tbl.add_column("Remote", style="dim", no_wrap=True) + tbl.add_column("Path", style="dim", overflow="fold") + for remote, path in skipped[:50]: + tbl.add_row(remote, path) + if len(skipped) > 50: + tbl.add_row("[dim]…[/]", f"[dim]+{len(skipped) - 50} more[/]") + console.print(tbl) + if variant_alerts: + console.print() + console.print(Panel( + f"[bold yellow]⚠ {len(variant_alerts)} variant alert(s) — manual review recommended[/]", + border_style="yellow")) + for bare_id, entries in sorted(variant_alerts.items()): + tbl = Table(title=f"[bold yellow][{bare_id}] — bare + variant coexist[/]", + title_justify="left", show_lines=False, border_style="yellow", expand=True) + tbl.add_column("ID", style="yellow", no_wrap=True) + tbl.add_column("Size", justify="right", no_wrap=True) + tbl.add_column("Path", overflow="fold") + for e in sorted(entries, key=lambda x: x.full_path): + eid = extract_id(Path(e.path).name) or e.jav_id + tbl.add_row(eid, human_size(e.size), e.full_path) + console.print(tbl) + console.print() + + +def decide_keep_with_reason(entries: list[FileEntry]) -> tuple[FileEntry, dict[str, str]]: + """Pick KEEP candidate and explain the first ranking rule that settled it. + + Catalog entries are excluded — they are offline/informational. + + Ranking (descending priority, configurable via keep_ranking in config.json): + 1. Video files in ordered priority folders outrank other rclone entries. + 2. Source entries outrank Target entries when no priority-folder video exists. + 3. Non-.ts files outrank .ts files when a duplicate group has both. + 4. Largest file size. If sizes are within size_tolerance_mib, treated as equal + and format preference is consulted instead. + 5. Format preference: ordered list of extensions (e.g. mkv > mp4 > wmv > avi). + 6. Tie-break: has resolution tag in filename ([1080p], [2160p], [720p], [480p]). + 7. Tie-break: longer filename (more metadata = more descriptive). + """ + ranking = _KEEP_RANKING or {} + tolerance_bytes = int(float(ranking.get("size_tolerance_mib") or 0) * 1024 * 1024) + priority_folders: list[str] = [ + str(folder).strip() for folder in + (ranking.get("priority_folders") or DEFAULT_KEEP_RANKING["priority_folders"]) + if str(folder).strip() + ] + fmt_order: list[str] = list( + ranking.get("format_preference") or DEFAULT_KEEP_RANKING["format_preference"] + ) + use_res_tag: bool = ranking.get("tiebreak_res_tag", True) + use_longer_name: bool = ranking.get("tiebreak_longer_name", True) + + rclone = [e for e in entries if e.source != "Catalog"] + + def _priority_folder_rank(e: FileEntry) -> int | None: + if Path(e.path).suffix.lower() not in _VIDEO_EXTS: + return None + # A root can be cq:JAV while the favored folder is a child path, or the + # supplied root can itself end in that folder. Match across full_path. + full_path = e.full_path.replace("\\", "/").strip("/").lower() + segments = [segment for segment in full_path.split("/") if segment] + for index, raw_folder in enumerate(priority_folders): + folder = raw_folder.replace("\\", "/").strip("/").lower() + if not folder: + continue + if "/" in folder or ":" in folder: + framed = f"/{full_path}/" + if full_path == folder or full_path.startswith(folder + "/") or f"/{folder}/" in framed: + return index + elif folder in segments: + return index + return None + + prioritized = [(rank, e) for e in rclone if (rank := _priority_folder_rank(e)) is not None] + best_priority = min((rank for rank, _ in prioritized), default=None) + priority_videos = [e for rank, e in prioritized if rank == best_priority] + pool_priority = [e for e in rclone if e.source == "Source"] + reason = {"code": "fallback", "summary": "First remaining duplicate candidate"} + if priority_videos: + pool = priority_videos + reason = { + "code": "vip_folder", + "summary": f"VIP folder: {priority_folders[best_priority]}", + } + elif pool_priority: + pool = pool_priority + reason = {"code": "source", "summary": "Source copy outranks target copies"} + else: + pool = rclone if rclone else entries + + # Transport streams often inflate size without being the better keeper. + preferred_containers = [ + e for e in pool if Path(e.path).suffix.lower() not in _LOWEST_KEEP_PRIORITY_EXTS + ] + if preferred_containers and len(preferred_containers) != len(pool): + pool = preferred_containers + reason = {"code": "container", "summary": "Non-TS video outranks transport stream"} + + # Step 1: narrow to within size tolerance of the maximum + max_size = max(e.size for e in pool) + candidates = [e for e in pool if max_size - e.size <= tolerance_bytes] + + if len(candidates) == 1: + if len(pool) > 1 and reason["code"] not in {"vip_folder", "source", "container"}: + reason = {"code": "size", "summary": "Largest file after ranking rules"} + return candidates[0], reason + + # Step 2: format preference (lower index in fmt_order = higher priority) + def _fmt_rank(e: FileEntry) -> int: + ext = Path(e.path).suffix.lower().lstrip(".") + try: + return fmt_order.index(ext) # lower = better + except ValueError: + return len(fmt_order) # unknown = lowest + + best_fmt = min(_fmt_rank(e) for e in candidates) + by_fmt = [e for e in candidates if _fmt_rank(e) == best_fmt] + if len(by_fmt) != len(candidates): + ext = Path(by_fmt[0].path).suffix.lower().lstrip(".").upper() or "preferred format" + reason = {"code": "format", "summary": f"Format preference: {ext}"} + candidates = by_fmt + + if len(candidates) == 1: + return candidates[0], reason + + # Step 3: resolution tag tie-break + if use_res_tag: + tagged = [e for e in candidates if _RES_LABEL_RE.search(Path(e.path).name)] + if tagged: + if len(tagged) != len(candidates): + reason = {"code": "resolution_tag", "summary": "Filename has a resolution tag"} + candidates = tagged + + if len(candidates) == 1: + return candidates[0], reason + + # Step 4: longer filename tie-break + if use_longer_name: + keep = max(candidates, key=lambda e: len(Path(e.path).name)) + return keep, {"code": "filename", "summary": "Longer filename tie-break"} + + return candidates[0], reason + + +def decide_keep(entries: list[FileEntry]) -> FileEntry: + """Pick KEEP candidate for duplicate output.""" + return decide_keep_with_reason(entries)[0] + + +def find_dupes(entries: Iterable[FileEntry]) -> dict[str, list[FileEntry]]: + """Group entries by jav_id. A group is a dupe only if it has >=2 non-Catalog entries.""" + groups: dict[str, list[FileEntry]] = {} + for e in entries: + # Re-evaluate duplicate keys from the current filename rules. Cached + # entries may predate a new part detector such as `.1of2`; treating those + # stale base IDs as duplicate files would produce risky delete hints. + key = extract_id(Path(e.path).name) or e.jav_id + groups.setdefault(key, []).append(e) + out: dict[str, list[FileEntry]] = {} + for k, v in groups.items(): + rclone_count = sum(1 for e in v if e.source != "Catalog") + if rclone_count >= 2: + out[k] = v + return out + + +_SUSPICIOUS_MULTIPART_TAIL_RE = re.compile( + r"(?:^|[-_.\s])(?:p|pt|part|cd|disc|ep|episode|vol|volume|scene)[-_.\s]*([a-d]|\d{1,2})(?:$|[-_.\s\[])" + r"|(?:^|[-_.\s])([a-d]|\d{1,2})(?:$|\s*\[)", + re.IGNORECASE, +) + + +def describe_dupe_risks(jav_id: str, entries: list[FileEntry]) -> list[dict[str, str]]: + """Flag duplicate groups that deserve manual review before deletion.""" + rclone = [e for e in entries if e.source != "Catalog"] + risks: list[dict[str, str]] = [] + if "#part" not in jav_id and len(rclone) >= 3: + risks.append({ + "code": "large_same_id_group", + "summary": f"{len(rclone)} files share this base ID; review for unrecognized parts before deleting.", + }) + + suspicious: list[str] = [] + for e in rclone: + stem = Path(e.path).stem + base_match = PRIMARY_ID_RE.match(stem) or COMPOUND_ID_RE.match(stem) or FALLBACK_ID_RE.match(stem) + if not base_match: + continue + tail = _RESOLUTION_TAG_RE.sub("", stem[base_match.end():]).strip() + if _SUSPICIOUS_MULTIPART_TAIL_RE.search(tail): + suspicious.append(Path(e.path).name) + if suspicious and "#part" not in jav_id: + samples = ", ".join(suspicious[:3]) + more = " ..." if len(suspicious) > 3 else "" + risks.append({ + "code": "part_like_suffix", + "summary": f"Part-like suffixes still share the base ID: {samples}{more}", + }) + return risks + + +def find_variant_alerts( + entries: Iterable[FileEntry], +) -> dict[str, list[FileEntry]]: + """Detect IDs where a bare form and a lowercase-variant form coexist. + + Example: both ``IBW-902.mp4`` and ``IBW-902z.mp4`` are present. + They are different products — not dupes — but their coexistence is + suspicious and warrants manual comparison. + + Returns {bare_id: [all entries whose re-evaluated ID matches bare or variant]}. + Only bare IDs that have at least one variant sibling are included. + """ + index: dict[str, list[FileEntry]] = {} + for e in entries: + key = extract_id(Path(e.path).name) or e.jav_id + index.setdefault(key, []).append(e) + + alerts: dict[str, list[FileEntry]] = {} + for jav_id in index: + if "#" in jav_id: + continue # skip multipart IDs + m = _VARIANT_SUFFIX_RE.match(jav_id) + if not m: + continue + bare = m.group(1) + if bare in index: + # Merge bare + variant entries under the bare key. + if bare not in alerts: + alerts[bare] = list(index[bare]) + alerts[bare].extend(index[jav_id]) + return alerts + + +# ---------- library issues (non-canonical filenames) ---------- + +def _bracket_to_canonical(filename: str) -> str: + """[REAL-779].mp4 → REAL-779.mp4 | [HODV-21076] Saki [1080p].mkv → HODV-21076 Saki [1080p].mkv""" + stem = Path(filename).stem + suffix = Path(filename).suffix + bm = _BRACKET_ID_RE.match(stem) + if not bm: + return filename + inner = bm.group(1).strip() + rest = stem[bm.end():].strip() + new_stem = f"{inner} {rest}".strip() if rest else inner + return f"{new_stem}{suffix}" + + +def _nohyphen_to_canonical(filename: str) -> str: + """MVSD312 [576p].avi → MVSD-312 [576p].avi""" + stem = Path(filename).stem + suffix = Path(filename).suffix + m = _NOHYPHEN_ID_RE.match(stem) + if not m: + return filename + prefix = m.group(1).upper() + num_str = m.group(2) + rest = stem[m.end():] + return f"{prefix}-{num_str}{rest}{suffix}" + + +def find_library_issues(cache: dict) -> dict: + """Scan cache for files with non-canonical names. + + Returns: + {"bracket_names": [...], "nohyphen_names": [...]} + Each entry: {remote, path, size, mod_time, jav_id, canonical_name, issue} + """ + bracket: list[dict] = [] + nohyphen: list[dict] = [] + for remote, remote_data in cache.get("remotes", {}).items(): + for f in remote_data.get("files", []): + fname = Path(f["path"]).name + stem = Path(fname).stem + if stem.startswith("[") and _BRACKET_ID_RE.match(stem): + bracket.append({ + "remote": remote, + "path": f["path"], + "size": f.get("size", 0), + "size_human": human_size(f.get("size", 0)), + "mod_time": f.get("mod_time", ""), + "jav_id": f.get("jav_id", ""), + "canonical_name": _bracket_to_canonical(fname), + "issue": "bracket_id", + }) + elif (not PRIMARY_ID_RE.match(stem) + and not COMPOUND_ID_RE.match(stem) + and not FALLBACK_ID_RE.match(stem) + and _NOHYPHEN_ID_RE.match(stem)): + nohyphen.append({ + "remote": remote, + "path": f["path"], + "size": f.get("size", 0), + "size_human": human_size(f.get("size", 0)), + "mod_time": f.get("mod_time", ""), + "jav_id": f.get("jav_id", ""), + "canonical_name": _nohyphen_to_canonical(fname), + "issue": "nohyphen_id", + }) + return {"bracket_names": bracket, "nohyphen_names": nohyphen} + + +def rename_file_in_remote( + remote: str, + old_rel_path: str, + new_rel_path: str, + cache: dict, + rclone_bin: str = "rclone", + save: bool = True, +) -> dict: + """Rename one file via rclone moveto and patch cache.json. + + Returns {"ok": True, "old_path": ..., "new_path": ...} + or {"ok": False, "error": ..., "conflict": bool} + + Pass save=False when batching — caller is responsible for calling save_cache() once. + """ + sep = "" if remote.endswith("/") else "/" + old_full = f"{remote}{sep}{old_rel_path}" + new_full = f"{remote}{sep}{new_rel_path}" + + # Collision check — does target already exist? + check = subprocess.run( + [rclone_bin, "lsf", new_full], + capture_output=True, text=True, + ) + if check.returncode == 0 and check.stdout.strip(): + return {"ok": False, "error": f"Target already exists: {new_full}", "conflict": True} + + # Perform rename + result = subprocess.run( + [rclone_bin, "moveto", old_full, new_full], + capture_output=True, text=True, + ) + if result.returncode != 0: + return {"ok": False, "error": (result.stderr or result.stdout).strip(), "conflict": False} + + # Patch cache — update path + jav_id for the renamed entry + remote_data = cache.get("remotes", {}).get(remote) + if remote_data: + for f in remote_data.get("files", []): + if f["path"] == old_rel_path: + f["path"] = new_rel_path + f["jav_id"] = extract_id(Path(new_rel_path).name) or f["jav_id"] + break + remote_data["skipped"] = [s for s in remote_data.get("skipped", []) if s != old_rel_path] + if save: + save_cache(cache) + + return {"ok": True, "old_path": old_full, "new_path": new_full} + + +def rename_files_batch( + renames: list[dict], + cache: dict, + rclone_bin: str = "rclone", +) -> list[dict]: + """Rename multiple files, writing cache once at the end. + + Each item in renames: {remote, old_path, new_path} + Returns list of per-file results with old_path/new_path echoed back. + """ + results = [] + cache_dirty = False + for r in renames: + res = rename_file_in_remote( + r["remote"], r["old_path"], r["new_path"], + cache, rclone_bin=rclone_bin, save=False, + ) + res["old_path"] = r["old_path"] + res["new_path"] = r["new_path"] + results.append(res) + if res["ok"]: + cache_dirty = True + if cache_dirty: + save_cache(cache) + return results + + +# ---------- plain renderers (--basic) ---------- + +def render_banner_plain(cache_meta: dict[str, dict], mode: str) -> str: + lines = [ansi(f"=== rc-jav ({mode}) ===", ANSI_BOLD)] + for r, m in cache_meta.items(): + if m["cached"]: + tag = f"CACHED {m['age']}" + (" STALE" if m["stale"] else "") + tag_c = ansi(tag, ANSI_YELLOW if m["stale"] else ANSI_DIM) + else: + tag_c = ansi("FRESH SCAN", ANSI_GREEN) + count_str = ansi(f"({m['file_count']} files)", ANSI_DIM) + lines.append(f" {r} {tag_c} {count_str}") + return "\n".join(lines) + + +def render_search_plain(matches: dict[str, list[FileEntry]], queries: list[str], + cache_meta: dict[str, dict]) -> str: + lines: list[str] = [] + if cache_meta: + lines.append(render_banner_plain(cache_meta, "search")) + lines.append("") + for q in queries: + hits = matches.get(q, []) + if not hits: + lines.append(ansi(f"[{q}] NOT FOUND", ANSI_RED)) + lines.append("") + continue + lines.append(ansi(f"[{q}] {len(hits)} hit(s)", ANSI_GREEN + ANSI_BOLD)) + for e in sorted(hits, key=lambda x: (x.jav_id, x.path.lower())): + meta = cache_meta.get(e.remote, {}) + if meta.get("cached"): + tag = ansi("[CACHED-STALE]", ANSI_YELLOW) if meta.get("stale") else ansi("[CACHED]", ANSI_DIM) + else: + tag = ansi("[FRESH]", ANSI_GREEN) + src = ansi(e.source, ANSI_YELLOW) + lines.append(f" {src} {tag}") + lines.append(ansi(f" file: {Path(e.path).name}", ANSI_BOLD)) + lines.append(f" size: {human_size(e.size)} ({e.size:,} bytes)") + lines.append(ansi(f" path: {e.full_path}", ANSI_DIM)) + lines.append("") + return "\n".join(lines) + + +# ---------- file outputs ---------- + +def render_dupes_plain(dupes, skipped, variant_alerts=None) -> str: + lines: list[str] = [] + if not dupes: + lines.append(ansi("No duplicates found.", ANSI_GREEN)) + else: + lines.append(ansi(f"Found {len(dupes)} duplicate ID group(s):", ANSI_BOLD)) + lines.append("") + total_reclaim = 0 + for jav_id in sorted(dupes): + entries = dupes[jav_id] + keep = decide_keep(entries) + lines.append(ansi(f"[{jav_id}]", ANSI_BOLD)) + for e in sorted(entries, key=lambda x: (x.source != "Source", x.source == "Catalog", -x.size)): + if e.source == "Catalog": + mark = ansi("CATALOG ", ANSI_CYAN) + elif e is keep: + mark = ansi("KEEP ", ANSI_GREEN) + else: + mark = ansi("DELETE? ", ANSI_RED) + total_reclaim += e.size + src = ansi(f"{e.source:>8}", ANSI_YELLOW) + size_str = f"{human_size(e.size)} ({e.size:,} B)" + lines.append(f" {mark} {src} {size_str:>26} {e.full_path}") + lines.append("") + lines.append(ansi(f"Potential space reclaim if all DELETE? removed: {human_size(total_reclaim)}", ANSI_BOLD)) + if skipped: + lines.append("") + lines.append(ansi(f"Skipped {len(skipped)} file(s) with no parseable ID:", ANSI_DIM)) + for remote, path in skipped[:50]: + lines.append(ansi(f" {remote} {path}", ANSI_DIM)) + if len(skipped) > 50: + lines.append(ansi(f" ... +{len(skipped) - 50} more", ANSI_DIM)) + if variant_alerts: + lines.append("") + lines.append(ansi(f"⚠ {len(variant_alerts)} variant alert(s) — manual review required:", ANSI_YELLOW + ANSI_BOLD)) + for bare_id, entries in sorted(variant_alerts.items()): + lines.append(ansi(f" [{bare_id}] bare + variant coexist", ANSI_YELLOW)) + for e in sorted(entries, key=lambda x: x.full_path): + eid = extract_id(Path(e.path).name) or e.jav_id + lines.append(f" {ansi(eid, ANSI_YELLOW)} {human_size(e.size):>10} {e.full_path}") + return "\n".join(lines) + + +def write_txt(path: Path, dupes, skipped): + path.write_text(render_dupes_plain(dupes, skipped), encoding="utf-8") + + +def write_csv(path: Path, dupes): + with path.open("w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(["jav_id", "action", "source", "remote", "path", "full_path", + "size_bytes", "size_human", "mod_time"]) + for jav_id in sorted(dupes): + entries = dupes[jav_id] + keep = decide_keep(entries) + for e in entries: + if e.source == "Catalog": + action = "CATALOG" + elif e is keep: + action = "KEEP" + else: + action = "DELETE?" + w.writerow([jav_id, action, e.source, + e.remote, e.path, e.full_path, e.size, human_size(e.size), e.mod_time]) + + +def describe_skipped_id(remote: str, path: str) -> dict[str, str]: + """Explain a common reason a path did not yield an ID.""" + name = Path((path or "").replace("\\", "/")).name + reason = "No supported JAV ID at filename start" + hint = "Rename with a leading ID such as ABC-123 or add an ID normalizer/site-specific source." + if re.match(r"^\[[A-Za-z0-9-]+-\d+\]", name): + reason = "ID is wrapped in leading brackets" + hint = "Remove the leading brackets so the filename starts with the ID." + elif re.match(r"^[A-Za-z][A-Za-z0-9]+[\u2010-\u2015]\d+", name): + reason = "ID uses a non-ASCII dash" + hint = "Replace the separator with a normal hyphen." + elif re.match(r"^[A-Za-z][A-Za-z0-9]+\d+", name): + reason = "ID prefix and number have no hyphen" + hint = "Insert the ID hyphen, for example ABC-123." + return {"remote": remote, "path": path, "name": name, "reason": reason, "hint": hint} + + +def dupes_to_obj(dupes, skipped, variant_alerts=None) -> dict: + out = {"groups": {}, "skipped": [describe_skipped_id(r, p) for r, p in skipped], + "variant_alerts": []} + for jav_id in sorted(dupes): + entries = dupes[jav_id] + keep, keep_reason = decide_keep_with_reason(entries) + out["groups"][jav_id] = { + "keep": asdict(keep) | {"full_path": keep.full_path, "size_human": human_size(keep.size)}, + "keep_reason": keep_reason, + "risks": describe_dupe_risks(jav_id, entries), + "delete_candidates": [asdict(e) | {"full_path": e.full_path, "size_human": human_size(e.size)} + for e in entries + if e is not keep and e.source != "Catalog"], + "catalog": [asdict(e) | {"full_path": e.full_path, "size_human": human_size(e.size)} + for e in entries if e.source == "Catalog"], + } + for bare_id, entries in sorted((variant_alerts or {}).items()): + out["variant_alerts"].append({ + "bare_id": bare_id, + "files": [ + asdict(e) | {"full_path": e.full_path, "size_human": human_size(e.size), + "detected_id": extract_id(Path(e.path).name) or e.jav_id} + for e in sorted(entries, key=lambda x: x.full_path) + ], + }) + return out + + +def write_json(path: Path, dupes, skipped, variant_alerts=None): + path.write_text(json.dumps(dupes_to_obj(dupes, skipped, variant_alerts), indent=2), encoding="utf-8") + + +# ---------- main ---------- + +def main(): + ap = argparse.ArgumentParser(description="Report duplicate JAV files across rclone remotes (read-only).") + ap.add_argument("--source", "-s", action="append", default=[], metavar="REMOTE", + help="Source remote path (priority — wins dupes regardless of size). Repeatable.") + ap.add_argument("--target", "-t", action="append", default=[], metavar="REMOTE", + help="Target remote path (non-priority — largest size wins among targets). Repeatable.") + ap.add_argument("--format", choices=["console", "txt", "csv", "json", "all"], + default="console") + ap.add_argument("--output-dir", default="./reports", help="Where to write txt/csv/json.") + ap.add_argument("--no-color", action="store_true") + ap.add_argument("--rclone-bin", default="rclone", + help="Path to rclone executable (default: 'rclone' on PATH).") + ap.add_argument("--search", action="append", default=[], metavar="ID", + help="Search mode: look up a JAV ID (e.g. SSIS-001). Repeatable. " + "If no --source/--target given, default target is used.") + ap.add_argument("--name", action="append", default=[], metavar="STR", + help="Substring/glob search against filename. Case-insensitive. " + "Repeatable; OR semantics (any token match = hit). " + "Supports * and ? wildcards. Use quotes for spaces.") + ap.add_argument("--update", "-u", action="store_true", + help="Search mode: force re-scan and overwrite cache for requested remotes.") + ap.add_argument("--no-cache", action="store_true", + help="Search mode: bypass cache entirely (no read, no write).") + ap.add_argument("--quick", "-q", action="store_true", + help="Force quick mode: skip cache, query rclone directly with --include glob. " + "Default is auto: single exact IDs use quick, wildcards/ranges/multi use cached.") + ap.add_argument("--cache", action="store_true", + help="Force cached mode (opposite of --quick).") + ap.add_argument("--save", action="store_true", + help="Persist the --source / --target / --catalog values you passed " + "as new defaults in config.json next to the script. " + "Only keys you explicitly passed are saved.") + ap.add_argument("--scan", action="store_true", + help="Walk configured remotes, refresh cache, exit. No search/dupe output. " + "Default scope: DEFAULT_TARGET. Override with --source/--target. " + "Always overwrites cache. Suitable for Task Scheduler / cron.") + ap.add_argument("--scan-since", metavar="DURATION", + help="Incremental scan: only walk files modified within DURATION " + "(e.g. 24h, 7d, 30m, 90s). Merges new/changed entries on top of " + "the existing cache; old entries are preserved. Falls back to a " + "full scan if there's no prior cache for a remote. Requires --scan.") + ap.add_argument("--catalog", action="append", default=[], metavar="PATH", + help="Path to a WinCatalog CSV or XML export. Repeatable. " + "Listed under 'Catalog' in results (informational, never KEEP/DELETE?).") + ap.add_argument("--part-pattern", action="append", default=[], metavar="REGEX", + help="Extra multipart filename regex. Repeatable; first capture group must be the part number. " + "Patterns run against the filename stem after built-in part detectors.") + ap.add_argument("--library-issues", action="store_true", + help="Report non-canonical filenames (bracket-wrapped IDs, no-hyphen IDs). " + "Reads from cache. Outputs JSON when --format json, plain otherwise.") + ap.add_argument("--rename-file", action="store_true", + help="Rename one file in a remote and patch cache. " + "Requires --remote, --old-path, --new-path. Outputs JSON.") + ap.add_argument("--rename-files-batch", action="store_true", + help="Rename multiple files in one call, writing cache once. " + "Reads JSON array of {remote, old_path, new_path} from stdin. Outputs JSON.") + ap.add_argument("--remote", metavar="REMOTE", + help="Remote path root for --rename-file (e.g. cq:JAV).") + ap.add_argument("--old-path", metavar="PATH", + help="Relative path of the file to rename (within --remote).") + ap.add_argument("--new-path", metavar="PATH", + help="New relative path after rename (within --remote).") + ap.add_argument("--basic", action="store_true", + help="Plain text output, no rich tables/panels/progress bars. " + "Useful for piping or low-bandwidth terminals.") + ap.add_argument("--clearjav", action="store_true", + help="Shortcut: use DEFAULT_SOURCE as --source and DEFAULT_TARGET as --target, " + "Equivalent to " + "`--source cq:personal-files/ClearJAV --target cq:personal-files/JAV/TMP`.") + args = ap.parse_args() + + global RCLONE_BIN, console, BASIC, DEFAULT_SOURCE, DEFAULT_TARGET, DEFAULT_CATALOG + RCLONE_BIN = args.rclone_bin + BASIC = args.basic or args.format == "json" + + # Apply persisted config overrides BEFORE defaults are consulted. + cfg = load_config() + if "default_source" in cfg: + DEFAULT_SOURCE = list(cfg["default_source"]) + if "default_target" in cfg: + DEFAULT_TARGET = list(cfg["default_target"]) + if "default_catalog" in cfg: + DEFAULT_CATALOG = list(cfg["default_catalog"]) + global _KEEP_RANKING + _KEEP_RANKING = cfg.get("keep_ranking") or {} + part_patterns = list(cfg.get("part_patterns") or []) + list(args.part_pattern) + pattern_errors = configure_part_patterns(part_patterns) + if pattern_errors: + for err in pattern_errors: + console.print(f"[red]invalid part pattern:[/] {err}") + sys.exit(2) + + # --save: persist explicitly-passed values, exit. + if args.save: + if not (args.source or args.target or args.catalog or args.part_pattern): + console.print("[red]--save needs at least one --source/--target/--catalog/--part-pattern value to persist.[/]") + sys.exit(2) + new_cfg = dict(cfg) + if args.source: + new_cfg["default_source"] = list(args.source) + if args.target: + new_cfg["default_target"] = list(args.target) + if args.catalog: + new_cfg["default_catalog"] = list(args.catalog) + if args.part_pattern: + new_cfg["part_patterns"] = list(args.part_pattern) + save_config(new_cfg) + console.print(f"[green]Saved to {CONFIG_PATH}:[/]") + for k in ("default_source", "default_target", "default_catalog", "part_patterns"): + if k in new_cfg: + console.print(f" {k} = {new_cfg[k]}") + sys.exit(0) + global USE_ANSI + USE_ANSI = not args.no_color + if args.no_color or BASIC: + console = Console(no_color=True, color_system=None, highlight=False) + + # Search mode: defaults kick in if no remotes specified. + if args.clearjav: + if not args.source: + args.source = list(DEFAULT_SOURCE) + if not args.target: + args.target = list(DEFAULT_TARGET) + + if args.search and not args.source and not args.target: + args.target = list(DEFAULT_TARGET) + + # --scan: default to DEFAULT_TARGET only, always overwrite cache. + if args.scan: + if not args.source and not args.target: + args.target = list(DEFAULT_TARGET) + args.update = True + + # Use default catalog(s) if user passed none. + if not args.catalog and DEFAULT_CATALOG: + args.catalog = list(DEFAULT_CATALOG) + + # --library-issues: read-only cache scan for non-canonical filenames. + if args.library_issues: + cache = load_cache() + issues = find_library_issues(cache) + if args.format == "json" or BASIC: + print(json.dumps({"ok": True, **issues})) + else: + bracket = issues["bracket_names"] + nohyphen = issues["nohyphen_names"] + total = len(bracket) + len(nohyphen) + if not total: + console.print(Panel("[bold green]No library issues found.[/]", title="Library Issues")) + else: + from rich.table import Table + t = Table(title=f"Library Issues ({total} file(s))", show_lines=True) + t.add_column("Issue", style="yellow", width=14) + t.add_column("Current Name") + t.add_column("Canonical Name", style="green") + t.add_column("Remote", style="dim") + for e in bracket: + t.add_row("bracket ID", Path(e["path"]).name, + e["canonical_name"], e["remote"]) + for e in nohyphen: + t.add_row("no hyphen", Path(e["path"]).name, + e["canonical_name"], e["remote"]) + console.print(t) + sys.exit(0) + + # --rename-files-batch: rename multiple files, single cache write. + if args.rename_files_batch: + try: + renames = json.loads(sys.stdin.read()) + except json.JSONDecodeError as e: + print(json.dumps({"ok": False, "error": f"Invalid JSON on stdin: {e}"})) + sys.exit(1) + if not isinstance(renames, list): + print(json.dumps({"ok": False, "error": "stdin must be a JSON array"})) + sys.exit(1) + cache = load_cache() + results = rename_files_batch(renames, cache, rclone_bin=RCLONE_BIN) + ok = any(r["ok"] for r in results) + print(json.dumps({"ok": ok, "results": results})) + sys.exit(0 if ok else 1) + + # --rename-file: rename one file in a remote and patch cache. + if args.rename_file: + if not args.remote or not args.old_path or not args.new_path: + ap.error("--rename-file requires --remote, --old-path, and --new-path.") + cache = load_cache() + result = rename_file_in_remote( + args.remote, args.old_path, args.new_path, cache, rclone_bin=RCLONE_BIN + ) + print(json.dumps(result)) + sys.exit(0 if result["ok"] else 1) + + if not args.source and not args.target and not args.catalog: + ap.error("Provide at least one --source, --target, or --catalog.") + + # Scan-only mode: walk remotes, write cache, summary, exit. + if args.scan: + scan_since = None + if args.scan_since: + scan_since = parse_duration(args.scan_since) + if not scan_since: + console.print(f"[red]invalid --scan-since value: {args.scan_since!r} " + f"(expected e.g. 24h, 7d, 30m, 90s)[/]") + sys.exit(2) + cache = load_cache() + cache_meta: dict[str, dict] = {} + skipped: list[tuple[str, str]] = [] + t0 = time.perf_counter() + if BASIC: + # `--scan` resolves its default target above. Report only the + # remotes that this scan will actually walk; falling back here to + # DEFAULT_SOURCE would resurrect retired source roots in job UI. + _all_remotes = list(args.source) + list(args.target) + sys.stderr.write("SCAN_START " + json.dumps({ + "remotes": _all_remotes, "total": len(_all_remotes), + }) + "\n") + sys.stderr.flush() + entries = (cached_collect(args.source, "Source", skipped, cache, + use_cache=not args.no_cache, force_update=True, + cache_meta=cache_meta, scan_since=scan_since) + + cached_collect(args.target, "Target", skipped, cache, + use_cache=not args.no_cache, force_update=True, + cache_meta=cache_meta, scan_since=scan_since)) + if not args.no_cache: + save_cache(cache) + elapsed = time.perf_counter() - t0 + if BASIC: + sys.stderr.write(f"Scan complete: {len(entries)} files in {elapsed:.2f}s\n") + sys.stderr.write(f"Cache: {CACHE_PATH}\n" if not args.no_cache + else "Cache: (skipped, --no-cache)\n") + else: + console.print(f"[bold green]Scan complete:[/] {len(entries)} files in {elapsed:.2f}s") + if not args.no_cache: + console.print(f"[dim]Cache: {CACHE_PATH}[/]") + else: + console.print("[dim]Cache: (skipped, --no-cache)[/]") + sys.exit(0) + + skipped: list[tuple[str, str]] = [] + t0 = time.perf_counter() + + if args.search or args.name: + search_timings: dict[str, int] = {} + # If --name was passed without explicit remotes, fall back to default target + # (catalog default already injected earlier; don't let it suppress remote defaulting). + if args.name and not args.search and not args.source and not args.target: + args.target = list(DEFAULT_TARGET) + # Substring name search can't be server-side filtered on most backends — cache wins. + # Only the ID search shape benefits from quick (server-side prefix glob). + if args.name and not args.quick: + mode, reason = "cached", "name substring search — cache is faster than rclone --include" + else: + combined = list(args.search) + list(args.name) + mode, reason = choose_search_mode(combined, args.quick, args.cache) + if BASIC: + sys.stderr.write(f"Mode: {mode} ({reason})\n") + else: + mode_color = "green" if mode == "quick" else "cyan" + console.print(f"[{mode_color}]Mode: {mode}[/] [dim]({reason})[/]") + + phase_t0 = time.perf_counter() + cache = load_cache() + search_timings["cache_load_ms"] = round((time.perf_counter() - phase_t0) * 1000) + use_cache = not args.no_cache and mode == "cached" + cache_meta: dict[str, dict] = {} + phase_t0 = time.perf_counter() + if mode == "quick": + all_patterns: list[str] = [] + for raw in args.search: + all_patterns.extend(query_to_include_patterns(raw)) + all_patterns.extend(name_to_include_patterns(args.name)) + entries = [] + for r in args.source: + cache_meta[r] = {"cached": False, "age": "quick", "stale": False, "file_count": 0} + got = quick_search_remote(r, "Source", all_patterns, skipped) + entries.extend(got) + cache_meta[r]["file_count"] = len(got) + for r in args.target: + cache_meta[r] = {"cached": False, "age": "quick", "stale": False, "file_count": 0} + got = quick_search_remote(r, "Target", all_patterns, skipped) + entries.extend(got) + cache_meta[r]["file_count"] = len(got) + else: + entries = (cached_collect(args.source, "Source", skipped, cache, + use_cache, args.update, cache_meta) + + cached_collect(args.target, "Target", skipped, cache, + use_cache, args.update, cache_meta)) + search_timings["entry_collect_ms"] = round((time.perf_counter() - phase_t0) * 1000) + # Load each catalog separately so cache_meta gets the per-catalog count + # (was global total — every catalog reported the sum across all). + catalog_entries: list[FileEntry] = [] + phase_t0 = time.perf_counter() + for cp_str in args.catalog: + for cp in _expand_catalog_paths([cp_str]): + ext = cp.suffix.lower() + if ext == ".csv": + one = load_catalog_csv(cp, skipped) + elif ext == ".xml": + one = load_catalog_xml(cp, skipped) + else: + console.print(f"[yellow]WARN: unknown catalog format '{ext}' for {cp}; skipping.[/]") + continue + catalog_entries.extend(one) + cache_meta[f"catalog:{cp.name}"] = { + "cached": False, "age": "loaded", "stale": False, + "file_count": len(one), + } + entries.extend(catalog_entries) + search_timings["catalog_load_ms"] = round((time.perf_counter() - phase_t0) * 1000) + if use_cache and args.update: + save_cache(cache) + else: + if args.cache and not args.no_cache: + cache = load_cache() + cache_meta: dict[str, dict] = {} + entries = (cached_collect(args.source, "Source", skipped, cache, + use_cache=True, force_update=False, + cache_meta=cache_meta) + + cached_collect(args.target, "Target", skipped, cache, + use_cache=True, force_update=False, + cache_meta=cache_meta)) + else: + remotes_by_label = ([("Source", r) for r in args.source] + + [("Target", r) for r in args.target]) + entries = collect_with_progress(remotes_by_label, skipped) + entries.extend(load_catalogs(args.catalog, skipped)) + + elapsed = time.perf_counter() - t0 + if BASIC: + sys.stderr.write(f"Scanned/loaded {len(entries)} file(s) in {elapsed:.2f}s\n") + else: + console.print(f"[dim]Scanned/loaded {len(entries)} file(s) in {elapsed:.2f}s[/]") + + if args.search or args.name: + # query_expansions: original_raw -> list of normalized IDs / wildcard patterns to look up + query_expansions: dict[str, list[str]] = {} + queries: list[str] = [] + for raw in args.search: + if RANGE_RE.search(raw): + expanded = expand_range(raw) or [] + normed: list[str] = [] + for r in expanded: + n = normalize_id(r) + if n: + normed.append(n) + if not normed: + console.print(f"[yellow]WARN: range '{raw}' produced no valid IDs.[/]") + continue + queries.append(raw) + query_expansions[raw] = normed + continue + if "*" in raw or "?" in raw: + q = raw.upper() + queries.append(q) + query_expansions[q] = [q] + continue + norm = normalize_id(raw) + if not norm: + console.print(f"[yellow]WARN: cannot parse '{raw}' as a JAV ID, skipping.[/]") + continue + # Use the raw (upper-cased) form for display so leading zeros are preserved + # (e.g. user types PRTD-027 — keep it, don't show PRTD-27). Lookup still uses + # the normalized form internally. + display = raw.upper() + queries.append(display) + query_expansions[display] = [norm] + phase_t0 = time.perf_counter() + index: dict[str, list[FileEntry]] = {} + for e in entries: + index.setdefault(e.jav_id, []).append(e) + search_timings["index_ms"] = round((time.perf_counter() - phase_t0) * 1000) + phase_t0 = time.perf_counter() + matches: dict[str, list[FileEntry]] = {} + match_traces: dict[str, dict[int, dict[str, str]]] = {} + for q in queries: + expansions = query_expansions.get(q, [q]) + hits: list[FileEntry] = [] + seen: set[int] = set() + traces: dict[int, dict[str, str]] = {} + + def add_hit(entry: FileEntry, matched_query: str) -> None: + key = id(entry) + if key in seen: + return + seen.add(key) + hits.append(entry) + traces[key] = describe_id_match(q, matched_query, entry.jav_id, len(expansions)) + + for sub in expansions: + if "*" in sub or "?" in sub: + pat = sub if "#PART" in sub.upper() else sub + "*" + for k, v in index.items(): + if fnmatch.fnmatchcase(k, pat): + for e in v: + add_hit(e, sub) + elif "#part" in sub: + for e in index.get(sub, []): + add_hit(e, sub) + else: + for e in index.get(sub, []): + add_hit(e, sub) + for k, v in index.items(): + if k.startswith(sub + "#part"): + for e in v: + add_hit(e, sub) + matches[q] = hits + match_traces[q] = traces + search_timings["match_ms"] = round((time.perf_counter() - phase_t0) * 1000) + if args.format == "json": + # Structured output for tools that consume search results (e.g. the rclonex + # Brave extension). Includes everything needed to drive a UI: per-query hits + # with source/remote/path/size/mod_time, plus name-match block + skipped. + name_hits_json: list[FileEntry] = [] + if args.name: + for e in entries: + if name_match(Path(e.path).stem, args.name): + name_hits_json.append(e) + out_obj = { + "queries": [ + { + "query": q, + "hits": [ + {"source": e.source, "remote": e.remote, "path": e.path, + "full_path": e.full_path, "size": e.size, + "size_human": human_size(e.size), + "mod_time": e.mod_time, "jav_id": e.jav_id, + **match_traces.get(q, {}).get(id(e), {})} + for e in sorted(matches.get(q, []), key=lambda x: (x.jav_id, x.path.lower())) + ], + } + for q in queries + ], + "name_matches": [ + {"source": e.source, "remote": e.remote, "path": e.path, + "full_path": e.full_path, "size": e.size, + "size_human": human_size(e.size), "mod_time": e.mod_time, + "jav_id": e.jav_id, "match_kind": "name", + "match_reason": "Filename search", "match_confidence": "broad", + "matched_query": ", ".join(args.name), "matched_id": e.jav_id} + for e in sorted(name_hits_json, key=lambda x: (x.jav_id, x.path.lower())) + ], + "name_tokens": list(args.name), + "cache_meta": cache_meta, + "skipped_count": len(skipped), + "elapsed_sec": round(time.perf_counter() - t0, 3), + "timings": search_timings, + } + print(json.dumps(out_obj)) + id_ok = (not queries) or all(matches.values()) + name_ok = (not args.name) or bool(name_hits_json) + sys.exit(0 if (id_ok and name_ok) else 1) + if queries: + if BASIC: + print(render_search_plain(matches, queries, cache_meta)) + else: + render_search(matches, queries, cache_meta) + # --name results as a separate block + name_hits: list[FileEntry] = [] + if args.name: + for e in entries: + if name_match(Path(e.path).stem, args.name): + name_hits.append(e) + if BASIC: + print(render_name_matches_plain(name_hits, args.name, cache_meta)) + else: + render_name_matches(name_hits, args.name, cache_meta) + # Exit code: 0 if every search query had hits AND name-search (if used) returned hits. + id_ok = (not queries) or all(matches.values()) + name_ok = (not args.name) or bool(name_hits) + sys.exit(0 if (id_ok and name_ok) else 1) + + dupes = find_dupes(entries) + variant_alerts = find_variant_alerts(entries) + if args.format == "json" and BASIC: + print(json.dumps(dupes_to_obj(dupes, skipped, variant_alerts))) + sys.exit(0) + if BASIC: + print(render_dupes_plain(dupes, skipped, variant_alerts)) + else: + render_dupes(dupes, skipped, variant_alerts) + + if args.format != "console": + out_dir = Path(args.output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + targets = {"txt", "csv", "json"} if args.format == "all" else {args.format} + if "txt" in targets: + write_txt(out_dir / f"dupes-{stamp}.txt", dupes, skipped) + if "csv" in targets: + write_csv(out_dir / f"dupes-{stamp}.csv", dupes) + if "json" in targets: + write_json(out_dir / f"dupes-{stamp}.json", dupes, skipped, variant_alerts) + console.print(f"[dim]Reports written to {out_dir}[/]") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + console.print("\n[yellow]Aborted by user (Ctrl+C). Cache not written for in-flight scans.[/]") + sys.exit(130) diff --git a/tests/test_rules.py b/tests/test_rules.py new file mode 100644 index 0000000..5e4c147 --- /dev/null +++ b/tests/test_rules.py @@ -0,0 +1,74 @@ +import importlib.util +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("rcjav_rules", ROOT / "rc-jav.py") +RCJAV = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = RCJAV +SPEC.loader.exec_module(RCJAV) + + +def entry(path, size=1_000, jav_id="TEST-001"): + return RCJAV.FileEntry( + source="Target", + remote="cq:JAV", + path=path, + size=size, + mod_time="", + jav_id=jav_id, + ) + + +class IdRuleTests(unittest.TestCase): + def test_builtin_multipart_shapes_keep_parts_distinct(self): + expected = { + "KV-118 - Aiba Reika_PART1.mp4": "KV-118#part1", + "KV-118_A.mp4": "KV-118#part1", + "OFJE-195-7 [480p].mp4": "OFJE-195#part7", + "ABC-027.mp4": "ABC-027", + } + for name, jav_id in expected.items(): + with self.subTest(name=name): + self.assertEqual(RCJAV.extract_id(name), jav_id) + + def test_multipart_files_do_not_form_base_duplicate_group(self): + files = [ + entry("KV-118 - Aiba Reika_PART1.mp4", jav_id="KV-118"), + entry("KV-118 - Aiba Reika_PART2.mp4", jav_id="KV-118"), + entry("KV-118 - Aiba Reika_PART3.mp4", jav_id="KV-118"), + ] + self.assertEqual(RCJAV.find_dupes(files), {}) + + def test_large_same_id_group_gets_manual_review_risk(self): + files = [ + entry("TEST-001 direct.mp4"), + entry("TEST-001 edit.mp4"), + entry("TEST-001 mirror.mp4"), + ] + risks = RCJAV.describe_dupe_risks("TEST-001", files) + self.assertIn("large_same_id_group", {risk["code"] for risk in risks}) + + +class KeepRankingTests(unittest.TestCase): + def test_vip_folder_beats_larger_non_vip_copy(self): + keep, reason = RCJAV.decide_keep_with_reason([ + entry("ClearJAV/TEST-001.mp4", size=2_000), + entry("Other/TEST-001 [1080p].mkv", size=9_000), + ]) + self.assertEqual(keep.path, "ClearJAV/TEST-001.mp4") + self.assertEqual(reason["code"], "vip_folder") + + def test_ts_loses_to_non_ts_even_when_larger(self): + keep, reason = RCJAV.decide_keep_with_reason([ + entry("Other/TEST-001.ts", size=9_000), + entry("Other/TEST-001.mp4", size=8_000), + ]) + self.assertEqual(keep.path, "Other/TEST-001.mp4") + self.assertEqual(reason["code"], "container") + + +if __name__ == "__main__": + unittest.main()