#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""ZEN Audit - CFS OJK collector v17. Extended financial fields and district extraction; local-only."""
import csv, json, re, sys, time, calendar, os
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from datetime import date
import requests
from bs4 import BeautifulSoup

HERE = Path(__file__).resolve().parent
CFS_LIST_URL = "https://cfs.ojk.go.id/cfs/Report.aspx?BankTypeCode=BPK&BankTypeName=BPR+Konvensional"
CFS_REPORT_URL = "https://cfs.ojk.go.id/cfs/ReportViewerForm.aspx"
REPORT_CODES = {
    "posisi_keuangan": "BPK-901-000001",
    "laba_rugi": "BPK-901-000002",
    "kualitas_aset": "BPK-901-000003",
    "komitmen_kontinjensi": "BPK-901-000004",
    "informasi_lainnya": "BPK-901-000005",
}
ALL_FIELDS = ["aset", "surat_berharga", "penempatan_bank_lain", "kredit_diberikan", "total_ekuitas",
              "agunan_diambil_alih", "tabungan", "deposito", "simpanan_bank_lain", "pinjaman_diterima",
              "jumlah_pendapatan_operasional", "jumlah_beban_operasional", "laba_tahun_berjalan",
              "kpmm", "cadangan_ppka", "npl_neto", "npl_gross", "roa", "bopo", "nim", "ldr", "cash_ratio"]
REPORT_FIELDS = {
    "posisi_keuangan": ["aset", "surat_berharga", "penempatan_bank_lain", "kredit_diberikan", "total_ekuitas",
                        "agunan_diambil_alih", "tabungan", "deposito", "simpanan_bank_lain", "pinjaman_diterima"],
    "laba_rugi": ["jumlah_pendapatan_operasional", "jumlah_beban_operasional", "laba_tahun_berjalan"],
    "kualitas_aset": ["kpmm", "cadangan_ppka", "npl_neto", "npl_gross", "roa", "bopo", "nim", "ldr", "cash_ratio"],
    "komitmen_kontinjensi": [],
    "informasi_lainnya": [],
}
# Only explicit messages indicating no report/data count as TIDAK ADA LAPORAN.
NO_REPORT_PATTERNS = [
    r"tidak ada (?:data|laporan)", r"data tidak tersedia", r"laporan tidak tersedia",
    r"laporan.{0,300}?untuk periode.{0,100}?tidak tersedia",
    r"(?:posisi keuangan|laba rugi|kualitas aset produktif|komitmen dan kontinjensi|informasi lainnya).{0,200}?tidak tersedia",
    r"no data available", r"no records found", r"data belum tersedia", r"belum tersedia untuk periode"
]

def read_config():
    cfg = {"timeout_seconds": 30, "delay_seconds": 0.05, "retry_attempts": 2, "retry_delay_seconds": 1, "max_workers": 3}
    p = HERE / "config.json"
    if p.exists(): cfg.update(json.loads(p.read_text(encoding="utf-8")))
    return cfg

def load_banks(path):
    if not path.exists(): raise FileNotFoundError(f"Tidak ditemukan: {path}")
    with path.open(encoding="utf-8-sig", newline="") as f:
        reader = csv.DictReader(f)
        if not reader.fieldnames or not {"kode_bpr", "nama_bpr"}.issubset(reader.fieldnames):
            raise ValueError("Header CSV wajib memuat kode_bpr,nama_bpr")
        rows = list(reader)
    result, seen = [], set()
    for r in rows:
        code, name = (r.get("kode_bpr") or "").strip(), (r.get("nama_bpr") or "").strip()
        if not code or not name: continue
        if code in seen: raise ValueError(f"Kode bank duplikat: {code}")
        seen.add(code)
        result.append({"kode_bpr": code, "nama_bpr": name,
                       "provinsi": (r.get("provinsi") or "Bali").strip(),
                       "jenis_bpr": (r.get("jenis_bpr") or "KONVENSIONAL").strip().upper()})
    return result

def clean_text(html):
    soup = BeautifulSoup(html, "html.parser")
    rows = []
    for tr in soup.find_all("tr"):
        cells = [" ".join(c.get_text(" ", strip=True).split()) for c in tr.find_all(["td", "th"])]
        cells = [c for c in cells if c]
        if cells: rows.append(" | ".join(cells))
    if rows: return "\n".join(rows), rows
    return " ".join(soup.get_text(" ", strip=True).split()), []

def norm(s): return re.sub(r"\s+", " ", str(s or "").replace("\xa0", " ")).strip().lower()

def parse_num(s):
    """Parse angka CFS, termasuk format rugi dalam tanda kurung: (4,270,609,315)."""
    raw = str(s or "").strip().replace("\xa0", "").replace(" ", "").replace("%", "")
    negative_by_parentheses = raw.startswith("(") and raw.endswith(")")
    if negative_by_parentheses:
        raw = raw[1:-1]
    raw = raw.replace(",", "")
    if re.fullmatch(r"-?\d+(?:\.\d+)?", raw):
        try:
            value = float(raw)
            return -abs(value) if negative_by_parentheses else value
        except ValueError:
            pass
    return None

def number_after_label(text, label_pattern):
    # CFS menyajikan rugi dengan tanda kurung, misalnya (4,270,609,315).
    number_pattern = r"(\(?-?\d[\d,]*(?:\.\d+)?\)?)"
    m = re.search(label_pattern + r"\s*(?:\|\s*)?" + number_pattern, text, flags=re.I)
    if not m:
        return None
    val = parse_num(m.group(1))
    return None if val is None else {"value": val, "label": m.group(0).strip(), "raw_value": m.group(1)}

def extract_money(text, field):
    patterns = {
        "aset": r"\bTotal\s+Aset\b", "surat_berharga": r"\bSurat\s+Berharga\b",
        "penempatan_bank_lain": r"\bPenempatan\s+pada\s+Bank\s+Lain\b",
        "total_ekuitas": r"\bTotal\s+Ekuitas\b", "agunan_diambil_alih": r"\bAgunan\s+yang\s+Diambil\s+Alih\b",
        "tabungan": r"^a\.?\s*Tabungan$|\bTabungan\b", "deposito": r"^b\.?\s*Deposito$|\bDeposito\b",
        "simpanan_bank_lain": r"\bSimpanan\s+dari\s+Bank\s+Lain\b", "pinjaman_diterima": r"\bPinjaman\s+yang\s+Diterima\b",
        "jumlah_pendapatan_operasional": r"\bJUMLAH\s+PENDAPATAN\s+OPERASIONAL\b",
        "jumlah_beban_operasional": r"\bJUMLAH\s+BEBAN\s+OPERASIONAL\b",
        "laba_tahun_berjalan": r"\bJUMLAH\s+LABA\s*\(RUGI\)\s+TAHUN\s+BERJALAN\b"}
    if field != "kredit_diberikan": return number_after_label(text, patterns[field])
    m = re.search(r"KREDIT\s+YANG\s+DIBERIKAN([\s\S]{0,5000}?)Penyertaan\s+Modal", text, flags=re.I)
    if m:
        matches = list(re.finditer(r"\bJumlah\b\s*(?:\|\s*)?(-?\d[\d,]*(?:\.\d+)?)", m.group(1), flags=re.I))
        if matches:
            x = matches[-1]
            return {"value": parse_num(x.group(1)), "label": x.group(0).strip(), "raw_value": x.group(1)}
    return None

RATIO_PATTERNS = {
    "kpmm": r"Kewajiban\s+Penyediaan\s+Modal\s+Minimum\s*\(KPMM\)",
    "cadangan_ppka": r"Rasio\s+Cadangan\s+terhadap\s+PPKA", "npl_neto": r"Non\s+Performing\s+Loan\s*\(NPL\)\s*Neto",
    "npl_gross": r"Non\s+Performing\s+Loan\s*\(NPL\)\s*Gross", "roa": r"Return\s+on\s+Assets\s*\(ROA\)",
    "bopo": r"Biaya\s+Operasional\s+terhadap\s+Pendapatan\s+Operasional\s*\(BOPO\)",
    "nim": r"Net\s+Interest\s+Margin\s*\(NIM\)", "ldr": r"Loan\s+to\s+Deposit\s+Ratio\s*\(LDR\)", "cash_ratio": r"Cash\s+Ratio"}

def make_session():
    s = requests.Session()
    s.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/130 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7"})
    return s

def resolve_cfs_bank_names(list_html, banks):
    """Use exact bank display names present in the CFS page when the HTML exposes them.
    Never guesses a bank code/name pairing; unmatched rows retain the CSV name and are logged.
    """
    soup = BeautifulSoup(list_html, "html.parser")
    options = []
    for opt in soup.find_all("option"):
        text = " ".join(opt.get_text(" ", strip=True).split())
        value = (opt.get("value") or "").strip()
        attrs = " ".join(str(v) for k, v in opt.attrs.items() if k != "value")
        options.append({"text": text, "value": value, "attrs": attrs})
    resolved, source = {}, {}
    for bank in banks:
        code = bank["kode_bpr"].strip()
        hits = []
        for opt in options:
            haystack = " ".join([opt["value"], opt["text"], opt["attrs"]])
            if re.search(r"(?<!\d)" + re.escape(code) + r"(?!\d)", haystack):
                if opt["text"] and opt["text"] not in ("- Pilih -", "Pilih Bank"):
                    hits.append(opt["text"])
        # Use a unique option label only. Ambiguous matches are deliberately ignored.
        hits = list(dict.fromkeys(hits))
        if len(hits) == 1:
            resolved[code] = hits[0]
            source[code] = "NAMA_RESMI_DARI_OPSI_CFS"
        else:
            resolved[code] = bank["nama_bpr"]
            source[code] = "NAMA_CSV_TIDAK_DAPAT_DICOCOKKAN_DI_HTML_CFS"
    return resolved, source

def explicit_no_report(html):
    text = norm(BeautifulSoup(html, "html.parser").get_text(" ", strip=True))
    return any(re.search(p, text, re.I) for p in NO_REPORT_PATTERNS)

def get_report(session, bank, month, year, report_code, timeout, referer):
    params = [("BankCodeNumber", bank["kode_bpr"]), ("BankCode", bank["nama_bpr"]), ("Month", str(month)),
              ("Year", str(year)), ("FinancialReportPeriodTypeCode", "R"), ("FinancialReportTypeCode", report_code)]
    r = session.get(CFS_REPORT_URL, params=params, headers={"Referer": referer}, timeout=timeout)
    if r.status_code >= 500:
        raise RuntimeError(f"ERROR_SERVER_HTTP_{r.status_code}")
    if r.status_code in (404, 410):
        return r.url, r.text, "TIDAK_ADA_LAPORAN", f"HTTP_{r.status_code}"
    r.raise_for_status()
    if explicit_no_report(r.text): return r.url, r.text, "TIDAK_ADA_LAPORAN", "Pesan eksplisit CFS menyatakan data/laporan tidak tersedia"
    if len(r.content) < 1000: raise RuntimeError(f"RESPONS_TIDAK_DAPAT_DIVERIFIKASI: hanya {len(r.content)} byte")
    if "Object reference not set to an instance of an object" in r.text:
        raise RuntimeError("ERROR_CFS_OBJECT_REFERENCE; tidak bisa disimpulkan sebagai laporan tidak tersedia")
    return r.url, r.text, "TERSEDIA", ""

def process_bank(bank, cfs_names, cfs_name_source, period, year, month, cfg):
    """Process one bank in an isolated session so concurrent workers don't share requests.Session."""
    session = make_session()
    resp = session.get(CFS_LIST_URL, timeout=int(cfg["timeout_seconds"]))
    resp.raise_for_status()
    referer = resp.url
    request_bank = {**bank, "nama_bpr": cfs_names.get(bank["kode_bpr"], bank["nama_bpr"])}
    rec = {**bank, "nama_bpr_cfs_digunakan": request_bank["nama_bpr"],
           "sumber_pencocokan_nama": cfs_name_source.get(bank["kode_bpr"], "NAMA_CSV"),
           "periode": date(year, month, calendar.monthrange(year, month)[1]).isoformat(),
           "status_verifikasi": "PERLU_DIVERIFIKASI", "status_bank_periode": "BELUM_DITENTUKAN",
           "status_laporan": {}, "sumber_laporan": {}, "indikator_ditemukan": {}, "peringatan": []}
    failures, diagnostics = [], []
    any_available, any_no_report, any_technical_error = False, False, False
    for report_name, code in REPORT_CODES.items():
        try:
            max_attempts = max(1, int(cfg.get("retry_attempts", 2)))
            for attempt in range(1, max_attempts + 1):
                try:
                    url, html, status, note = get_report(session, request_bank, month, year, code, int(cfg["timeout_seconds"]), referer)
                    break
                except Exception as attempt_error:
                    if attempt >= max_attempts:
                        raise
                    time.sleep(max(0.2, float(cfg.get("retry_delay_seconds", 1))))
            rec["status_laporan"][report_name] = {"status": status, "catatan": note}
            rec["sumber_laporan"][report_name] = url
            if status == "TIDAK_ADA_LAPORAN":
                any_no_report = True
                for field in REPORT_FIELDS.get(report_name, []):
                    if rec.get(field) is None:
                        rec[field] = 0
                        rec["indikator_ditemukan"][field] = {"laporan": report_name, "status_nilai": "LAPORAN_TIDAK_TERSEDIA_DIISI_0"}
                diagnostics.append({"kode_bpr": bank["kode_bpr"], "laporan": report_name, "status": status,
                    "catatan": note, "komponen_diisi_nol": REPORT_FIELDS.get(report_name, [])})
            else:
                text, rows = clean_text(html); extracted = {}
                # Ambil kabupaten/kota dari header laporan CFS, misalnya "Provinsi Bali, Kab. Tabanan".
                if not rec.get("kota_kabupaten"):
                    dm = re.search(r"(?:Provinsi\s+[^,|]+,\s*)?(Kab\.?|Kota)\s+([A-Za-zÀ-ÿ .'-]+)", text, flags=re.I)
                    if dm:
                        wilayah = "Kabupaten" if dm.group(1).lower().startswith("kab") else "Kota"
                        rec["kota_kabupaten"] = f"{wilayah} {dm.group(2).strip().splitlines()[0]}".strip()
                if report_name == "posisi_keuangan":
                    for field in REPORT_FIELDS[report_name]: extracted[field] = extract_money(text, field)
                elif report_name == "laba_rugi":
                    for field in REPORT_FIELDS[report_name]: extracted[field] = extract_money(text, field)
                elif report_name == "kualitas_aset":
                    for field, pattern in RATIO_PATTERNS.items(): extracted[field] = number_after_label(text, pattern)
                if REPORT_FIELDS.get(report_name) and not any(v is not None for v in extracted.values()):
                    raise RuntimeError("HALAMAN_VIEWER_TANPA_INDIKATOR: HTTP berhasil tetapi tidak ada indikator laporan yang terbaca; perlu verifikasi manual, tidak diisi nol")
                any_available = True
                for field, found in extracted.items():
                    if found is not None and rec.get(field) is None:
                        rec[field] = found["value"]
                        rec["indikator_ditemukan"][field] = {"laporan": report_name, "label": found["label"], "sel_nilai": found["raw_value"]}
                    elif found is None and rec.get(field) is None:
                        rec[field] = 0
                        rec["indikator_ditemukan"][field] = {"laporan": report_name, "status_nilai": "INDIKATOR_TIDAK_DITEMUKAN_DIISI_0"}
                diagnostics.append({"kode_bpr": bank["kode_bpr"], "laporan": report_name, "status": status,
                    "jumlah_baris": len(rows), "contoh_baris": rows[:5],
                    "indikator_ditemukan": [k for k,v in extracted.items() if v is not None],
                    "indikator_tidak_ditemukan_diisi_nol": [k for k,v in extracted.items() if v is None]})
            time.sleep(max(0, float(cfg.get("delay_seconds", 0.05))))
        except Exception as e:
            err = str(e)[:500]; any_technical_error = True
            rec["status_laporan"][report_name] = {"status": "GAGAL_DITARIK", "catatan": err}
            failures.append({"kode_bpr": bank["kode_bpr"], "nama_bpr": bank["nama_bpr"], "periode": period,
                "report": report_name, "status": "GAGAL_DITARIK", "error": err})
    if any_technical_error:
        rec["status_bank_periode"] = "GAGAL_DITARIK_SEBAGIAN_ATAU_SELURUHNYA"
    elif any_available and any_no_report:
        rec["status_bank_periode"] = "LAPORAN_SEBAGIAN_TIDAK_TERSEDIA"
    elif any_available:
        rec["status_bank_periode"] = "LAPORAN_TERSEDIA"
    elif any_no_report:
        rec["status_bank_periode"] = "TIDAK_ADA_LAPORAN"
    else:
        rec["status_bank_periode"] = "PERLU_DIVERIFIKASI"
    rec["indikator_kosong"] = [f for f in ALL_FIELDS if rec.get(f) is None]
    rec["sumber_url"] = "; ".join(rec["sumber_laporan"].values())
    rec["peringatan"] = ["Hasil otomatis wajib dicocokkan dengan laporan Excel/PDF resmi OJK.",
        "Komponen dari laporan yang tidak tersedia dan indikator yang tidak ditemukan pada laporan berhasil ditarik diisi 0.",
        "HTTP 500/Object reference/kegagalan koneksi ditandai GAGAL_DITARIK, bukan TIDAK ADA LAPORAN; komponen terkait tetap kosong jika belum dapat diverifikasi.",
        "TIDAK ADA LAPORAN diberikan jika CFS memberi indikasi eksplisit atau HTTP 404/410.",
        "Jika sebagian jenis laporan tersedia dan sebagian tidak, status bank-periode adalah LAPORAN_SEBAGIAN_TIDAK_TERSEDIA.",
        "Jika semua jenis laporan secara eksplisit tidak tersedia, seluruh 22 komponen yang dipetakan diisi 0 dan status bank-periode TIDAK_ADA_LAPORAN.",
        "Nama bank dicocokkan ke opsi HTML CFS bila kode bank dapat ditemukan secara unik; bila tidak, nama CSV digunakan dan dicatat.",
        "HTTP 500 dicoba ulang sesuai retry_attempts; jika tetap gagal, nilai terkait dibiarkan kosong dan status teknis tetap terlihat.",
        "Status LAPORAN_TERSEDIA tidak berarti seluruh indikator telah tervalidasi.",
        "v17: menambah surat berharga, agunan diambil alih, simpanan dari bank lain, pinjaman diterima, total pendapatan dan beban operasional, serta kabupaten/kota dari header CFS. v15: membaca angka negatif bertanda kurung. v12: mengenali pesan eksplisit CFS tidak tersedia. v13: pemrosesan paralel terbatas; setiap worker memakai sesi terpisah."]
    return rec, failures, diagnostics

def main():
    if len(sys.argv) < 3:
        print("Cara pakai: py collector_lengkap_v14.py YYYY-MM banks_template.csv"); return 2
    period = sys.argv[1].strip()
    if not re.fullmatch(r"\d{4}-(03|06|09|12)", period):
        print("Periode harus YYYY-03, YYYY-06, YYYY-09, atau YYYY-12."); return 2
    year, month = map(int, period.split("-")); bank_file = Path(sys.argv[2])
    if not bank_file.is_absolute(): bank_file = HERE / bank_file
    try: cfg, banks = read_config(), load_banks(bank_file)
    except Exception as e: print(f"Kesalahan konfigurasi/CSV: {e}"); return 2
    if not banks: print("Daftar bank kosong."); return 2
    session = make_session()
    try:
        resp = session.get(CFS_LIST_URL, timeout=int(cfg["timeout_seconds"])); resp.raise_for_status()
        cfs_names, cfs_name_source = resolve_cfs_bank_names(resp.text, banks)
        print(f"Halaman CFS: HTTP {resp.status_code}; cookie sesi: {len(session.cookies)}")
        mapped = sum(1 for b in banks if cfs_name_source[b["kode_bpr"]] == "NAMA_RESMI_DARI_OPSI_CFS")
        print(f"Pencocokan nama bank dari opsi CFS: {mapped}/{len(banks)}")
        for b in banks:
            if cfs_name_source[b["kode_bpr"]] != "NAMA_RESMI_DARI_OPSI_CFS":
                print(f"   Nama CFS belum terverifikasi untuk kode {b['kode_bpr']}; memakai nama CSV: {b['nama_bpr']}")
    except Exception as e: print(f"Gagal membuka CFS: {e}"); return 1
    workers = max(1, min(int(cfg.get("max_workers", 8)), 8, len(banks)))
    print(f"Mode cepat: {workers} BPR diproses bersamaan; jeda antar laporan {cfg.get('delay_seconds', 0.05)} detik.", flush=True)
    results = {}
    with ThreadPoolExecutor(max_workers=workers) as pool:
        futures = {pool.submit(process_bank, bank, cfs_names, cfs_name_source, period, year, month, cfg): (i, bank)
                   for i, bank in enumerate(banks, 1)}
        done_count = 0
        for fut in as_completed(futures):
            i, bank = futures[fut]
            try:
                rec, fails, diags = fut.result()
                results[i] = (rec, fails, diags)
                done_count += 1
                filled = sum(rec.get(f) is not None for f in ALL_FIELDS)
                print(f"[{done_count}/{len(banks)} selesai] {bank['nama_bpr']}: {rec['status_bank_periode']}; indikator {filled}/22", flush=True)
            except Exception as e:
                # Unexpected worker-level failure: record it, never turn it into zeros.
                rec = {**bank, "nama_bpr_cfs_digunakan": cfs_names.get(bank['kode_bpr'], bank['nama_bpr']),
                    "sumber_pencocokan_nama": cfs_name_source.get(bank['kode_bpr'], 'NAMA_CSV'),
                    "periode": date(year, month, calendar.monthrange(year, month)[1]).isoformat(),
                    "status_verifikasi": "PERLU_DIVERIFIKASI", "status_bank_periode": "GAGAL_DITARIK_SEBAGIAN_ATAU_SELURUHNYA",
                    "status_laporan": {}, "sumber_laporan": {}, "indikator_ditemukan": {},
                    "indikator_kosong": ALL_FIELDS.copy(), "peringatan": [str(e)], "sumber_url": ""}
                fail = [{"kode_bpr": bank['kode_bpr'], "nama_bpr": bank['nama_bpr'], "periode": period,
                         "report": "worker", "status": "GAGAL_DITARIK", "error": str(e)[:500]}]
                results[i] = (rec, fail, [])
                done_count += 1
                print(f"[{done_count}/{len(banks)} selesai] {bank['nama_bpr']}: GAGAL_DITARIK ({str(e)[:120]})", flush=True)
    records, failures, diagnostics = [], [], []
    for i in sorted(results):
        rec, fails, diags = results[i]
        records.append(rec); failures.extend(fails); diagnostics.extend(diags)
    out = {"records": records, "metadata": {"periode": period, "jumlah_bank": len(records), "mode": "LOCAL_ONLY",
        "max_workers": workers, "catatan": "Mode lokal saja. Pemrosesan paralel terbatas. Laporan yang secara eksplisit tidak tersedia diisi 0; kegagalan teknis atau halaman viewer tanpa indikator dibedakan dan tidak otomatis diisi 0."}}
    (HERE / "hasil_batch.json").write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
    (HERE / "gagal_batch.json").write_text(json.dumps(failures, ensure_ascii=False, indent=2), encoding="utf-8")
    (HERE / "diagnostik_batch.json").write_text(json.dumps(diagnostics, ensure_ascii=False, indent=2), encoding="utf-8")
    print("\nSELESAI - MODE LOKAL SAJA")
    print(f"Hasil: {HERE/'hasil_batch.json'}\nKegagalan: {HERE/'gagal_batch.json'}\nDiagnostik: {HERE/'diagnostik_batch.json'}")
    print(f"Laporan gagal: {len(failures)}; bank: {len(records)}")
    return 0
if __name__ == "__main__": raise SystemExit(main())