import asyncio
import aiohttp
import json
import logging
import os
import re
import signal
import sys
import time
from datetime import datetime
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

SETTINGS = {
    "delete_dir_name": "delete",
    "logs_dir_name": "logs",
    "pid_file_name": "spider.pid",
    "state_file_name": "spider_state.json",
    "input_file": "urls.txt",
    "output_file": "vystup.txt",

    "max_parallel": 2,
    "max_string_urls": 2,

    "log_level": "DEBUG",
    "state_write_every": 25,

    "request_retries": 1,
    "retry_backoff_seconds": 2.0,

    "timeout_total": 60,
    "timeout_connect": 24,
    "timeout_sock_connect": 24,
    "timeout_sock_read": 42,

    "max_response_bytes": 3_000_000,

    "contact_strings": [
        "contac"
    ],

    "skip_extensions": {
        ".pdf", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg",
        ".zip", ".rar", ".7z", ".tar", ".gz",
        ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
        ".mp3", ".mp4", ".avi", ".mov", ".wmv", ".webm",
        ".css", ".js", ".json", ".xml"
    },
}

DELETE_DIR = os.path.join(BASE_DIR, SETTINGS["delete_dir_name"])
LOG_DIR = os.path.join(DELETE_DIR, SETTINGS["logs_dir_name"])
PID_FILE = os.path.join(DELETE_DIR, SETTINGS["pid_file_name"])
STATE_FILE = os.path.join(DELETE_DIR, SETTINGS["state_file_name"])


def ensure_runtime_dirs():
    os.makedirs(LOG_DIR, exist_ok=True)


def setup_logging():
    ensure_runtime_dirs()
    logger = logging.getLogger("spider")
    logger.handlers = []
    logger.setLevel(getattr(logging, SETTINGS["log_level"].upper(), logging.INFO))
    logger.propagate = False

    formatter = logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
    log_file = os.path.join(
        LOG_DIR,
        f"spider_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.log",
    )

    fh = logging.FileHandler(log_file, encoding="utf-8")
    fh.setFormatter(formatter)
    logger.addHandler(fh)

    logger.info("Log initialized: %s", log_file)
    return logger


def write_state(data):
    ensure_runtime_dirs()
    tmp_file = STATE_FILE + ".tmp"
    with open(tmp_file, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)
    os.replace(tmp_file, STATE_FILE)


def is_process_running(pid):
    try:
        os.kill(pid, 0)
        return True
    except OSError:
        return False


def read_pid():
    if not os.path.exists(PID_FILE):
        return None
    try:
        with open(PID_FILE, "r", encoding="utf-8") as f:
            return int(f.read().strip())
    except Exception:
        return None


def write_pid():
    ensure_runtime_dirs()
    with open(PID_FILE, "w", encoding="utf-8") as f:
        f.write(str(os.getpid()))


def remove_pid():
    if os.path.exists(PID_FILE):
        os.remove(PID_FILE)


def stop_running_instance():
    pid = read_pid()
    if not pid:
        return 0

    if not is_process_running(pid):
        remove_pid()
        return 0

    try:
        os.kill(pid, signal.SIGTERM)
    except OSError:
        remove_pid()
        return 0

    for _ in range(20):
        if not is_process_running(pid):
            remove_pid()
            return 0
        time.sleep(0.2)

    try:
        os.kill(pid, signal.SIGKILL)
    except OSError:
        pass

    remove_pid()
    return 0


def sanitize_url(url: str) -> str:
    return (url or "").split("|", 1)[0].strip()


def parse_input_line(line):
    line = sanitize_url(line)
    if line.startswith(("http://", "https://")):
        return line
    return None


def normalize_domain(url):
    netloc = urlparse(url).netloc.lower()
    if netloc.startswith("www."):
        netloc = netloc[4:]
    return netloc


def same_domain(url_a, url_b):
    return normalize_domain(url_a) == normalize_domain(url_b)


def contains_string(value, strings):
    value = (value or "").lower()
    return any(s in value for s in strings)


def clean_text(value):
    return (value or "").replace("|", " ").replace("\n", " ").replace("\r", " ").strip()


def should_skip_url(url):
    path = urlparse(url).path.lower()
    return any(path.endswith(ext) for ext in SETTINGS["skip_extensions"])


def format_result_line(row):
    return (
        f"{row['url']}|{row['kontaktni_url']}|{row['title']}|{row['meta_desc']}|"
        f"{row['meta_kw']}|{row['ma_kontakt']}|{row['emails']}|{row['phones']}|{row['reason']}\n"
    )


def load_processed_urls(output_file):
    processed = set()
    if not os.path.exists(output_file):
        return processed

    with open(output_file, "r", encoding="utf-8", errors="ignore") as f:
        for line in f:
            line = line.rstrip("\n")
            if not line:
                continue
            pos = line.find("|")
            if pos > 0:
                processed.add(line[:pos])
    return processed


def extract_emails(html: str):
    email_pattern = r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}"
    raw = re.findall(email_pattern, html)
    out = []

    banned_domain_suffixes = {
        ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".svg",
        ".css", ".js", ".json", ".xml", ".woff", ".woff2", ".ttf",
        ".ico", ".bmp"
    }

    for e in raw:
        e = e.strip().lower()

        if "/" in e or "\\" in e:
            continue

        parts = e.split("@", 1)
        if len(parts) != 2:
            continue

        local, domain = parts
        if not local or not domain:
            continue

        if any(domain.endswith(ext) for ext in banned_domain_suffixes):
            continue

        if re.search(r"@\d+x\d+\.", e):
            continue

        if re.search(r"_\d+x\d+@", e):
            continue

        if local.endswith(("_80x53", "_150x", "_300x", "_600x", "_1200x")):
            continue

        out.append(e)

    return list(dict.fromkeys(out))[:10]


def extract_phones(html: str):
    tel_pattern = r"(?:\+|00)?\d[\d\s\-\(\)\.\/]{6,20}\d"
    raw = re.findall(tel_pattern, html)
    out = []

    for tel in raw:
        tel = tel.strip()
        digits = re.sub(r"\D", "", tel)

        if len(digits) < 7 or len(digits) > 15:
            continue

        if re.fullmatch(r"(19|20)\d{2}", digits):
            continue

        if re.fullmatch(r"(19|20)\d{2}(0[1-9]|1[0-2])([0-2]\d|3[01])", digits):
            continue

        if re.fullmatch(r"\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4}", tel):
            continue

        if re.fullmatch(r"\d{4}[-/.]\d{1,2}[-/.]\d{1,2}", tel):
            continue

        if tel.isdigit() and len(digits) >= 9:
            continue

        if re.fullmatch(r"\d{5,}-\d{1,4}", tel):
            continue

        if digits.startswith("16") and len(digits) > 10:
            continue

        out.append(tel)

    return list(dict.fromkeys(out))[:10]


async def fetch_html(session, url, logger, stage):
    start_ts = time.monotonic()

    if should_skip_url(url):
        logger.debug("FETCH_SKIP stage=%s url=%s reason=skip_extension", stage, url)
        return "", 0, "", "skip_extension", 0.0

    for attempt in range(SETTINGS["request_retries"] + 1):
        attempt_no = attempt + 1
        logger.debug("FETCH_START stage=%s attempt=%s url=%s", stage, attempt_no, url)

        try:
            async with session.get(url, allow_redirects=True) as resp:
                status = resp.status
                final_url = str(resp.url)
                content_type = (resp.headers.get("Content-Type") or "").lower()

                if status >= 400:
                    elapsed = time.monotonic() - start_ts
                    logger.debug(
                        "FETCH_HTTP_ERROR stage=%s url=%s final_url=%s status=%s ctype=%s elapsed=%.2f",
                        stage, url, final_url, status, content_type, elapsed,
                    )
                    return "", status, content_type, f"http_{status}", elapsed

                if "text/html" not in content_type and "application/xhtml+xml" not in content_type:
                    elapsed = time.monotonic() - start_ts
                    logger.debug(
                        "FETCH_NON_HTML stage=%s url=%s final_url=%s status=%s ctype=%s elapsed=%.2f",
                        stage, url, final_url, status, content_type, elapsed,
                    )
                    return "", status, content_type, "non_html", elapsed

                body = await resp.read()
                if not body:
                    elapsed = time.monotonic() - start_ts
                    logger.debug(
                        "FETCH_EMPTY_BODY stage=%s url=%s final_url=%s status=%s ctype=%s elapsed=%.2f",
                        stage, url, final_url, status, content_type, elapsed,
                    )
                    return "", status, content_type, "empty_body", elapsed

                raw_len = len(body)
                body = body[: SETTINGS["max_response_bytes"]]
                text = body.decode("utf-8", errors="ignore")
                elapsed = time.monotonic() - start_ts

                logger.debug(
                    "FETCH_OK stage=%s url=%s final_url=%s status=%s ctype=%s bytes=%s elapsed=%.2f",
                    stage, url, final_url, status, content_type, raw_len, elapsed,
                )
                return text, status, content_type, "ok", elapsed

        except asyncio.TimeoutError:
            elapsed = time.monotonic() - start_ts
            logger.debug(
                "FETCH_TIMEOUT stage=%s attempt=%s url=%s elapsed=%.2f",
                stage, attempt_no, url, elapsed,
            )
            if attempt < SETTINGS["request_retries"]:
                await asyncio.sleep(SETTINGS["retry_backoff_seconds"] * attempt_no)
            else:
                return "", 0, "", "timeout", elapsed

        except aiohttp.ClientConnectorDNSError:
            elapsed = time.monotonic() - start_ts
            logger.debug(
                "FETCH_DNS_ERROR stage=%s attempt=%s url=%s elapsed=%.2f",
                stage, attempt_no, url, elapsed,
            )
            return "", 0, "", "dns_error", elapsed

        except aiohttp.ClientError as e:
            elapsed = time.monotonic() - start_ts
            logger.debug(
                "FETCH_CLIENT_ERROR stage=%s attempt=%s url=%s exc=%s elapsed=%.2f",
                stage, attempt_no, url, type(e).__name__, elapsed,
            )
            if attempt < SETTINGS["request_retries"]:
                await asyncio.sleep(SETTINGS["retry_backoff_seconds"] * attempt_no)
            else:
                return "", 0, "", f"client_error:{type(e).__name__}", elapsed

        except Exception as e:
            elapsed = time.monotonic() - start_ts
            logger.debug(
                "FETCH_EXCEPTION stage=%s attempt=%s url=%s exc=%s elapsed=%.2f",
                stage, attempt_no, url, type(e).__name__, elapsed,
            )
            return "", 0, "", f"exception:{type(e).__name__}", elapsed

    elapsed = time.monotonic() - start_ts
    return "", 0, "", "unknown_fetch_failure", elapsed


async def find_contact_links(soup, base_url, strings, logger):
    found = []

    for link in soup.find_all("a", href=True):
        href = (link.get("href") or "").lower()
        text = link.get_text(" ", strip=True).lower()

        if contains_string(href, strings) or contains_string(text, strings):
            try:
                full_url = urljoin(base_url, link["href"])
                found.append(full_url)
            except Exception:
                pass

    dedup = []
    seen = set()
    for url in found:
        if url not in seen:
            seen.add(url)
            dedup.append(url)

    logger.debug(
        "CONTACT_LINK_SCAN base=%s found_count=%s links=%s",
        base_url, len(dedup), dedup[:20],
    )
    return dedup


async def parse_page(url, html, strings, logger, source_label):
    soup = BeautifulSoup(html, "html.parser")
    contact_links = await find_contact_links(soup, url, strings, logger)
    kontaktni_url = contact_links[0] if contact_links else ""

    title = soup.title.string.strip() if soup.title and soup.title.string else ""

    meta_desc = ""
    meta_kw = ""
    for meta in soup.find_all("meta"):
        name = (meta.get("name") or "").lower()
        if name == "description":
            meta_desc = meta.get("content", "")
        elif name == "keywords":
            meta_kw = meta.get("content", "")

    emails = extract_emails(html)
    phones = extract_phones(html)

    logger.debug(
        "PARSE_RESULT source=%s url=%s title_len=%s meta_desc_len=%s meta_kw_len=%s emails=%s phones=%s kontaktni_url=%s",
        source_label,
        url,
        len(title or ""),
        len(meta_desc or ""),
        len(meta_kw or ""),
        len(emails),
        len(phones),
        kontaktni_url,
    )

    return {
        "url": url,
        "kontaktni_url": kontaktni_url or url,
        "title": clean_text(title),
        "meta_desc": clean_text(meta_desc),
        "meta_kw": clean_text(meta_kw),
        "ma_kontakt": bool(emails),
        "emails": "; ".join(emails),
        "phones": "; ".join(phones),
        "reason": "contact_found" if emails else "no_contact_found",
        "contact_links": contact_links,
    }


async def process_one(url, session, logger, strings):
    started = time.monotonic()
    logger.debug("PROCESS_START url=%s", url)

    try:
        html0, status0, ctype0, reason0, elapsed0 = await fetch_html(
            session, url, logger, "homepage"
        )

        if not html0:
            logger.debug(
                "PROCESS_END url=%s result=no_homepage_html reason=%s status=%s ctype=%s homepage_elapsed=%.2f total_elapsed=%.2f",
                url, reason0, status0, ctype0, elapsed0, time.monotonic() - started,
            )
            return {
                "url": url,
                "kontaktni_url": "",
                "title": "",
                "meta_desc": "",
                "meta_kw": "",
                "ma_kontakt": False,
                "emails": "",
                "phones": "",
                "reason": f"homepage_{reason0}",
            }

        base_parsed = await parse_page(url, html0, strings, logger, "homepage")

        title = base_parsed["title"]
        meta_desc = base_parsed["meta_desc"]
        meta_kw = base_parsed["meta_kw"]
        kontaktni_url = base_parsed["kontaktni_url"] or ""

        all_emails = []
        all_phones = []

        if base_parsed["emails"]:
            all_emails.extend([x.strip() for x in base_parsed["emails"].split(";") if x.strip()])

        if base_parsed["phones"]:
            all_phones.extend([x.strip() for x in base_parsed["phones"].split(";") if x.strip()])

        candidates = []
        for candidate in base_parsed["contact_links"]:
            if not candidate.startswith(("http://", "https://")):
                continue
            if not same_domain(url, candidate):
                continue
            if should_skip_url(candidate):
                continue
            candidates.append(candidate)

        dedup_candidates = []
        seen = set()
        for candidate in candidates:
            if candidate not in seen:
                seen.add(candidate)
                dedup_candidates.append(candidate)
            if len(dedup_candidates) >= SETTINGS["max_string_urls"]:
                break

        logger.debug(
            "CANDIDATES url=%s homepage_status=%s homepage_ctype=%s homepage_reason=%s candidate_count=%s candidates=%s",
            url, status0, ctype0, reason0, len(dedup_candidates), dedup_candidates,
        )

        for idx, target in enumerate(dedup_candidates, start=1):
            logger.debug("CONTACT_FETCH_START url=%s candidate_index=%s candidate=%s", url, idx, target)

            html_t, status_t, ctype_t, reason_t, elapsed_t = await fetch_html(
                session, target, logger, f"contact_{idx}"
            )

            logger.debug(
                "CONTACT_FETCH_END url=%s candidate_index=%s candidate=%s status=%s ctype=%s reason=%s elapsed=%.2f",
                url, idx, target, status_t, ctype_t, reason_t, elapsed_t,
            )

            if not html_t:
                continue

            parsed = await parse_page(target, html_t, strings, logger, f"contact_{idx}")

            if parsed["emails"]:
                found = [x.strip() for x in parsed["emails"].split(";") if x.strip()]
                all_emails.extend(found)
                logger.debug(
                    "CONTACT_EMAILS_FOUND url=%s candidate=%s count=%s emails=%s",
                    url, target, len(found), found,
                )

            if parsed["phones"]:
                found_phones = [x.strip() for x in parsed["phones"].split(";") if x.strip()]
                all_phones.extend(found_phones)
                logger.debug(
                    "CONTACT_PHONES_FOUND url=%s candidate=%s count=%s phones=%s",
                    url, target, len(found_phones), found_phones,
                )

            if (not kontaktni_url) and parsed["kontaktni_url"]:
                kontaktni_url = parsed["kontaktni_url"]

        unique_emails = list(dict.fromkeys(all_emails))[:10]
        unique_phones = list(dict.fromkeys(all_phones))[:10]
        ma_kontakt = bool(unique_emails)

        logger.debug(
            "PROCESS_SUMMARY url=%s title_len=%s meta_desc_len=%s meta_kw_len=%s emails=%s phones=%s kontaktni_url=%s total_elapsed=%.2f",
            url,
            len(title or ""),
            len(meta_desc or ""),
            len(meta_kw or ""),
            len(unique_emails),
            len(unique_phones),
            kontaktni_url,
            time.monotonic() - started,
        )

        return {
            "url": url,
            "kontaktni_url": kontaktni_url or url,
            "title": title,
            "meta_desc": meta_desc,
            "meta_kw": meta_kw,
            "ma_kontakt": ma_kontakt,
            "emails": "; ".join(unique_emails),
            "phones": "; ".join(unique_phones),
            "reason": "contact_found" if ma_kontakt else "no_contact_found",
        }

    except Exception as e:
        logger.exception("PROCESS_FATAL url=%s exc=%s", url, type(e).__name__)
        return {
            "url": url,
            "kontaktni_url": "",
            "title": "",
            "meta_desc": "",
            "meta_kw": "",
            "ma_kontakt": False,
            "emails": "",
            "phones": "",
            "reason": f"fatal:{type(e).__name__}",
        }


async def worker(queue, session, logger, strings, write_lock, stats):
    while True:
        url = await queue.get()
        if url is None:
            queue.task_done()
            return

        result = await process_one(url, session, logger, strings)

        async with write_lock:
            with open(SETTINGS["output_file"], "a", encoding="utf-8") as f:
                f.write(format_result_line(result))
                f.flush()

            stats["processed"] += 1
            if result["emails"]:
                stats["contacts"] += 1

            logger.debug(
                "WRITE_RESULT processed=%s total=%s url=%s reason=%s contact_present=%s",
                stats["processed"],
                stats["total_urls"],
                result["url"],
                result["reason"],
                bool(result["emails"]),
            )

            if stats["processed"] % SETTINGS["state_write_every"] == 0:
                write_state({
                    "status": "running",
                    "started_at": stats["started_at"],
                    "total_urls": stats["total_urls"],
                    "processed_urls": stats["processed"],
                    "total_contacts": stats["contacts"],
                })
                logger.info(
                    "PROGRESS processed=%s/%s contacts=%s",
                    stats["processed"],
                    stats["total_urls"],
                    stats["contacts"],
                )

        queue.task_done()


async def main():
    logger = setup_logging()
    strings = SETTINGS["contact_strings"]

    logger.info("Pouzite strings: %s", ", ".join(strings))
    logger.info(
        "Timeouts total=%s connect=%s sock_connect=%s sock_read=%s max_parallel=%s max_string_urls=%s retries=%s backoff=%s",
        SETTINGS["timeout_total"],
        SETTINGS["timeout_connect"],
        SETTINGS["timeout_sock_connect"],
        SETTINGS["timeout_sock_read"],
        SETTINGS["max_parallel"],
        SETTINGS["max_string_urls"],
        SETTINGS["request_retries"],
        SETTINGS["retry_backoff_seconds"],
    )

    if not os.path.exists(SETTINGS["input_file"]):
        logger.info("Vstupni soubor neexistuje: %s", SETTINGS["input_file"])
        return 1

    processed_urls = load_processed_urls(SETTINGS["output_file"])
    if processed_urls:
        logger.info("Resume: preskakuji jiz zapsanych URL: %s", len(processed_urls))

    urls = []
    bad = 0
    with open(SETTINGS["input_file"], "r", encoding="utf-8") as f:
        for line in f:
            parsed = parse_input_line(line)
            if parsed:
                if parsed not in processed_urls:
                    urls.append(parsed)
            elif line.strip():
                bad += 1

    if bad:
        logger.info("Preskoceno neplatnych radku: %s", bad)

    logger.info("Do fronty zarazeno %s URL", len(urls))

    stats = {
        "started_at": datetime.now().isoformat(),
        "total_urls": len(urls),
        "processed": 0,
        "contacts": 0,
    }

    write_state({
        "status": "running",
        "started_at": stats["started_at"],
        "total_urls": len(urls),
        "processed_urls": 0,
        "total_contacts": 0,
    })

    timeout = aiohttp.ClientTimeout(
        total=SETTINGS["timeout_total"],
        connect=SETTINGS["timeout_connect"],
        sock_connect=SETTINGS["timeout_sock_connect"],
        sock_read=SETTINGS["timeout_sock_read"],
    )

    connector = aiohttp.TCPConnector(
        limit=SETTINGS["max_parallel"],
        ttl_dns_cache=300,
    )

    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
        "Connection": "keep-alive",
    }

    queue = asyncio.Queue()
    for url in urls:
        await queue.put(url)
    for _ in range(SETTINGS["max_parallel"]):
        await queue.put(None)

    write_lock = asyncio.Lock()

    async with aiohttp.ClientSession(timeout=timeout, connector=connector, headers=headers) as session:
        tasks = [
            asyncio.create_task(worker(queue, session, logger, strings, write_lock, stats))
            for _ in range(SETTINGS["max_parallel"])
        ]
        await queue.join()
        await asyncio.gather(*tasks)

    write_state({
        "status": "finished",
        "finished_at": datetime.now().isoformat(),
        "total_urls": stats["total_urls"],
        "processed_urls": stats["processed"],
        "total_contacts": stats["contacts"],
    })

    logger.info("Hotovo! processed=%s contacts=%s", stats["processed"], stats["contacts"])
    return 0


def run():
    if len(sys.argv) > 1 and sys.argv[1] == "--stop":
        return stop_running_instance()

    existing_pid = read_pid()
    if existing_pid and is_process_running(existing_pid):
        return 1

    write_pid()
    try:
        return asyncio.run(main())
    finally:
        remove_pid()


if __name__ == "__main__":
    raise SystemExit(run())