some fix for nude save db with
This commit is contained in:
parent
eeed9af545
commit
9f8f767ae5
1 changed files with 40 additions and 8 deletions
|
|
@ -1,8 +1,9 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import random
|
import random
|
||||||
from pathlib import PurePosixPath
|
from pathlib import Path, PurePosixPath
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
|
@ -35,6 +36,37 @@ REQUEST_HEADERS = {
|
||||||
_NUDE_HISTORY_LOCK = asyncio.Lock()
|
_NUDE_HISTORY_LOCK = asyncio.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_int_setting(name: str, default: int) -> int:
|
||||||
|
value = getattr(config, name, None)
|
||||||
|
if value is None:
|
||||||
|
value = os.getenv(name, str(default))
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
logger.warning("Invalid %s=%r, using default %s", name, value, default)
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_path_setting(name: str, default: str) -> Path:
|
||||||
|
value = getattr(config, name, None)
|
||||||
|
if value is None:
|
||||||
|
value = os.getenv(name, default)
|
||||||
|
|
||||||
|
if isinstance(value, Path):
|
||||||
|
return value
|
||||||
|
|
||||||
|
return Path(str(value))
|
||||||
|
|
||||||
|
|
||||||
|
NUDE_HISTORY_PATH = _resolve_path_setting("NUDE_HISTORY_PATH", "/db/nude_history.json")
|
||||||
|
NUDE_POSTS_PER_PAGE = _resolve_int_setting("NUDE_POSTS_PER_PAGE", 100)
|
||||||
|
NUDE_POSTS_POOL_TARGET = _resolve_int_setting("NUDE_POSTS_POOL_TARGET", 1200)
|
||||||
|
NUDE_MAX_POSTS_TO_SCAN = _resolve_int_setting("NUDE_MAX_POSTS_TO_SCAN", 2000)
|
||||||
|
NUDE_MIN_UNSEEN_POOL = _resolve_int_setting("NUDE_MIN_UNSEEN_POOL", 100)
|
||||||
|
NUDE_HISTORY_MAX_SIZE = _resolve_int_setting("NUDE_HISTORY_MAX_SIZE", 5000)
|
||||||
|
|
||||||
|
|
||||||
def _extract_photo_candidates(posts: list[dict]) -> list[dict]:
|
def _extract_photo_candidates(posts: list[dict]) -> list[dict]:
|
||||||
candidates = []
|
candidates = []
|
||||||
seen_urls = set()
|
seen_urls = set()
|
||||||
|
|
@ -81,7 +113,7 @@ def _guess_filename(image_url: str) -> str:
|
||||||
|
|
||||||
|
|
||||||
def _load_nude_history() -> dict:
|
def _load_nude_history() -> dict:
|
||||||
path = config.NUDE_HISTORY_PATH
|
path = NUDE_HISTORY_PATH
|
||||||
if not path.exists():
|
if not path.exists():
|
||||||
return {"sent_images": []}
|
return {"sent_images": []}
|
||||||
|
|
||||||
|
|
@ -100,11 +132,11 @@ def _load_nude_history() -> dict:
|
||||||
sent_images = []
|
sent_images = []
|
||||||
|
|
||||||
cleaned = [item for item in sent_images if isinstance(item, str) and item.strip()]
|
cleaned = [item for item in sent_images if isinstance(item, str) and item.strip()]
|
||||||
return {"sent_images": cleaned[-config.NUDE_HISTORY_MAX_SIZE :]}
|
return {"sent_images": cleaned[-NUDE_HISTORY_MAX_SIZE :]}
|
||||||
|
|
||||||
|
|
||||||
def _save_nude_history(data: dict) -> None:
|
def _save_nude_history(data: dict) -> None:
|
||||||
path = config.NUDE_HISTORY_PATH
|
path = NUDE_HISTORY_PATH
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
tmp_path = path.with_name(f"{path.name}.tmp")
|
tmp_path = path.with_name(f"{path.name}.tmp")
|
||||||
with tmp_path.open("w", encoding="utf-8") as fh:
|
with tmp_path.open("w", encoding="utf-8") as fh:
|
||||||
|
|
@ -119,7 +151,7 @@ def _remember_sent_image(history: dict, image_url: str) -> None:
|
||||||
|
|
||||||
sent_images = [item for item in sent_images if item != image_url]
|
sent_images = [item for item in sent_images if item != image_url]
|
||||||
sent_images.append(image_url)
|
sent_images.append(image_url)
|
||||||
history["sent_images"] = sent_images[-config.NUDE_HISTORY_MAX_SIZE :]
|
history["sent_images"] = sent_images[-NUDE_HISTORY_MAX_SIZE :]
|
||||||
|
|
||||||
|
|
||||||
async def _fetch_posts_page(
|
async def _fetch_posts_page(
|
||||||
|
|
@ -161,8 +193,8 @@ async def _fetch_posts_page(
|
||||||
async def _fetch_latest_photo_candidates(
|
async def _fetch_latest_photo_candidates(
|
||||||
session: aiohttp.ClientSession, sent_urls: set[str]
|
session: aiohttp.ClientSession, sent_urls: set[str]
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
per_page = max(1, min(config.NUDE_POSTS_PER_PAGE, 100))
|
per_page = max(1, min(NUDE_POSTS_PER_PAGE, 100))
|
||||||
max_scan = max(config.NUDE_POSTS_POOL_TARGET, config.NUDE_MAX_POSTS_TO_SCAN)
|
max_scan = max(NUDE_POSTS_POOL_TARGET, NUDE_MAX_POSTS_TO_SCAN)
|
||||||
max_pages = max(1, (max_scan + per_page - 1) // per_page)
|
max_pages = max(1, (max_scan + per_page - 1) // per_page)
|
||||||
|
|
||||||
candidates = []
|
candidates = []
|
||||||
|
|
@ -191,7 +223,7 @@ async def _fetch_latest_photo_candidates(
|
||||||
if image_url not in sent_urls:
|
if image_url not in sent_urls:
|
||||||
unseen_count += 1
|
unseen_count += 1
|
||||||
|
|
||||||
if len(candidates) >= config.NUDE_POSTS_POOL_TARGET and unseen_count >= config.NUDE_MIN_UNSEEN_POOL:
|
if len(candidates) >= NUDE_POSTS_POOL_TARGET and unseen_count >= NUDE_MIN_UNSEEN_POOL:
|
||||||
break
|
break
|
||||||
if len(candidates) >= max_scan:
|
if len(candidates) >= max_scan:
|
||||||
break
|
break
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue