import json, re, html as htmlmod, sys, time
from urllib.parse import urlparse
import requests
from xml.etree import ElementTree as ET
from bs4 import BeautifulSoup

BASE = 'https://www.krb.nsw.edu.au'
FEED = BASE + '/principals-blog/feed/'
UA = {'User-Agent': 'Mozilla/5.0 (compatible; IGNITE-migration/1.0)'}
OUT = '/Users/iggy/.hermes/profiles/ignite_team/outbound/krb_principals_blog_items.json'

def clean_text(s):
    if s is None:
        return ''
    s = htmlmod.unescape(str(s))
    s = re.sub(r'\s+', ' ', s).strip()
    return s

def slug_from_url(url):
    path = urlparse(url).path.strip('/')
    parts = path.split('/')
    return parts[-1] if parts else ''

def parse_feed_page(page):
    url = FEED if page == 1 else FEED + f'?paged={page}'
    r = requests.get(url, headers=UA, timeout=30)
    if r.status_code != 200:
        return []
    try:
        root = ET.fromstring(r.content)
    except Exception as e:
        print(f'feed xml error page {page}: {e}', file=sys.stderr)
        return []
    items=[]
    for item in root.findall('./channel/item'):
        title = clean_text(item.findtext('title'))
        link = clean_text(item.findtext('link'))
        pub = clean_text(item.findtext('pubDate'))
        desc = clean_text(item.findtext('description'))
        creator = clean_text(item.findtext('{http://purl.org/dc/elements/1.1/}creator'))
        guid = clean_text(item.findtext('guid'))
        wp_id = None
        m = re.search(r'[?&]p=(\d+)', guid)
        if m:
            wp_id = int(m.group(1))
        items.append({'feed_page':page,'title':title,'link':link,'pubDate':pub,'description':desc,'creator':creator,'guid':guid,'wp_id':wp_id})
    return items

def sanitize_body(entry):
    # Remove UI/embedded cruft but keep editorial HTML.
    for tag in entry.find_all(['script','style','noscript','iframe','form','button']):
        tag.decompose()
    # unwrap spans/font and empty editor artifacts
    for tag in entry.find_all(['span','font']):
        tag.unwrap()
    allowed = {'p','h2','h3','h4','h5','h6','ul','ol','li','blockquote','strong','b','em','i','a','br','hr','img','figure','figcaption','table','thead','tbody','tr','th','td'}
    for tag in entry.find_all(True):
        if tag.name not in allowed:
            tag.unwrap()
            continue
        attrs = {}
        if tag.name == 'a' and tag.get('href'):
            attrs['href'] = tag.get('href')
            if tag.get('target'):
                attrs['target'] = tag.get('target')
        if tag.name == 'img' and tag.get('src'):
            attrs['src'] = tag.get('src')
            if tag.get('alt'):
                attrs['alt'] = tag.get('alt')
        tag.attrs = attrs
    out = ''.join(str(child) for child in entry.children).strip()
    out = re.sub(r'\n\s*\n+', '\n', out)
    return out

def scrape_post(feed_item):
    url = feed_item['link']
    r = requests.get(url, headers=UA, timeout=30)
    r.raise_for_status()
    soup = BeautifulSoup(r.text, 'lxml')
    h1 = soup.select_one('h1.entry-title') or soup.select_one('h1')
    title = clean_text(h1.get_text(' ', strip=True) if h1 else feed_item.get('title'))
    if not title:
        # fall back to og:title, then URL slug
        og = soup.find('meta', property='og:title')
        title = clean_text(og.get('content') if og else '') or slug_from_url(url).replace('-', ' ').title()
    slug = slug_from_url(url)
    entry = soup.select_one('div.entry-content')
    body = sanitize_body(entry) if entry else ''
    summary = clean_text(feed_item.get('description'))
    if not summary:
        meta = soup.find('meta', attrs={'name':'description'}) or soup.find('meta', property='og:description')
        summary = clean_text(meta.get('content') if meta else '')
    time_tag = soup.select_one('time.updated[datetime]')
    iso_date = time_tag.get('datetime') if time_tag and time_tag.get('datetime') else ''
    return {
        'name': title,
        'slug': slug,
        'post-summary': summary,
        'post-body': body,
        'published-date': iso_date,
        'source_url': url,
        'wp_id': feed_item.get('wp_id'),
        'feed_pubDate': feed_item.get('pubDate'),
        'feed_title': feed_item.get('title'),
        'author': feed_item.get('creator'),
        'body_chars': len(body),
        'summary_chars': len(summary),
    }

seen_links=set(); feed_items=[]
for page in range(1, 50):
    items = parse_feed_page(page)
    if not items:
        print(f'No items at feed page {page}; stopping')
        break
    new=0
    for it in items:
        if it['link'] and it['link'] not in seen_links:
            seen_links.add(it['link']); feed_items.append(it); new += 1
    print(f'Feed page {page}: {len(items)} items, {new} new')
    if new == 0:
        print('No new links; stopping')
        break

posts=[]; errors=[]
for i,it in enumerate(feed_items,1):
    try:
        post=scrape_post(it)
        posts.append(post)
        print(f'{i}/{len(feed_items)} ok {post["slug"]} title={post["name"]!r} body={post["body_chars"]} summary={post["summary_chars"]}')
    except Exception as e:
        errors.append({'link':it.get('link'),'error':repr(e)})
        print(f'{i}/{len(feed_items)} ERROR {it.get("link")}: {e}', file=sys.stderr)
    time.sleep(0.1)

with open(OUT,'w',encoding='utf-8') as f:
    json.dump({'count':len(posts),'errors':errors,'posts':posts}, f, ensure_ascii=False, indent=2)
print(json.dumps({'out':OUT,'feed_count':len(feed_items),'post_count':len(posts),'error_count':len(errors),'first':posts[0]['name'] if posts else None,'last':posts[-1]['name'] if posts else None}, indent=2))