"""
GameTora Umamusume Support Card Scraper
Scrapes all support cards with their effects at key levels (1, 25, 40, 50)
Includes character art download
"""

import sqlite3
import time
import re
import os
import requests
import sys
import os

# Ensure we can import from parent directory
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from playwright.sync_api import sync_playwright
from db.db_queries import get_conn, init_database

BASE_URL = "https://gametora.com"

if getattr(sys, 'frozen', False):
    # In frozen state, look in the same directory as the executable
    IMAGES_PATH = os.path.join(os.path.dirname(sys.executable), "images")
else:
    IMAGES_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "images")

# Key levels to scrape (limit break milestones)
KEY_LEVELS = [20, 25, 30, 35, 40, 45, 50]

# Rarity mapping based on image filename
RARITY_MAP = {
    "rarity_01": "R",
    "rarity_02": "SR", 
    "rarity_03": "SSR"
}

# Type mapping based on image filename - uses utx_ico_obtain_XX pattern!
TYPE_MAP = {
    "obtain_00": "Speed",
    "obtain_01": "Stamina",
    "obtain_02": "Power",
    "obtain_03": "Guts",
    "obtain_04": "Wisdom",
    "obtain_05": "Friend",
    "obtain_06": "Group"
}

def scrape_all_support_links(page):
    """Scrape all support card links from the list page"""
    print("Loading support card list...")
    page.goto(f"{BASE_URL}/umamusume/supports", timeout=60000)
    page.wait_for_load_state("networkidle")
    
    # Tick "Show Upcoming Supports" checkbox if it exists
    print("Checking for 'Show Upcoming Supports' checkbox...")
    checkbox_ticked = page.evaluate("""
        () => {
            // Look for checkbox or toggle related to "upcoming" or "future" supports
            const labels = Array.from(document.querySelectorAll('label, span, div'));
            const checkboxLabel = labels.find(el => 
                el.textContent.toLowerCase().includes('upcoming') && 
                el.textContent.toLowerCase().includes('support')
            );
            
            if (checkboxLabel) {
                // Try to find associated checkbox/input
                let checkbox = checkboxLabel.querySelector('input[type="checkbox"]');
                if (!checkbox) {
                    // Look for checkbox nearby
                    const parent = checkboxLabel.closest('div, label');
                    if (parent) {
                        checkbox = parent.querySelector('input[type="checkbox"]');
                    }
                }
                
                if (checkbox && !checkbox.checked) {
                    checkbox.click();
                    return true;
                }
            }
            
            // Alternative: Look for any checkbox with "upcoming" in nearby text
            const checkboxes = Array.from(document.querySelectorAll('input[type="checkbox"]'));
            for (const cb of checkboxes) {
                const text = cb.closest('label, div')?.textContent?.toLowerCase() || '';
                if (text.includes('upcoming') && text.includes('support') && !cb.checked) {
                    cb.click();
                    return true;
                }
            }
            
            return false;
        }
    """)
    
    if checkbox_ticked:
        print("  ✓ Ticked 'Show Upcoming Supports' checkbox")
        page.wait_for_timeout(1000)  # Wait for page to update
    else:
        print("  ℹ 'Show Upcoming Supports' checkbox not found or already ticked")
    
    # Scroll to load all cards (lazy loading) - more scrolls for complete list
    print("Scrolling to load all cards...")
    for i in range(30):
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        page.wait_for_timeout(400)
    
    # Scroll back up and down to ensure all loaded
    page.evaluate("window.scrollTo(0, 0)")
    page.wait_for_timeout(500)
    for i in range(10):
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        page.wait_for_timeout(300)
    
    # Extract all card links
    links = page.evaluate("""
        () => {
            const links = Array.from(document.querySelectorAll('a[href*="/umamusume/supports/"]'))
                .map(a => a.href)
                .filter(href => href.match(/\\/supports\\/\\d+-/));
            return [...new Set(links)];
        }
    """)
    
    print(f"Found {len(links)} support cards")
    return sorted(links)

def parse_rarity_from_image(img_src):
    """Extract rarity from image source URL"""
    if not img_src:
        return "R"  # Default to R if not found
    for key, rarity in RARITY_MAP.items():
        if key in img_src:
            return rarity
    return "R"  # Default to R

def parse_type_from_image(img_src):
    """Extract type from image source URL"""
    if not img_src:
        return "Unknown"
    for key, card_type in TYPE_MAP.items():
        if key in img_src:
            return card_type
    return "Unknown"

def get_max_level_for_rarity(rarity):
    """Get maximum level based on rarity"""
    if rarity == "SSR":
        return 50
    elif rarity == "SR":
        return 45
    else:  # R
        return 40

def extract_stable_id_from_url(url):
    """Extract stable numeric ID from GameTora URL (e.g., 30022 from /supports/30022-mejiro-mcqueen)"""
    match = re.search(r'/supports/(\d+)-', url)
    if match:
        return match.group(1)
    return None

def download_card_image(page, stable_id, card_name):
    """Download the card's character art image"""
    os.makedirs(IMAGES_PATH, exist_ok=True)
    
    try:
        # Find the main card image
        img_url = page.evaluate("""
            () => {
                // Look for the main card image - usually a large character portrait
                const imgs = Array.from(document.querySelectorAll('img'));
                
                // Find images that might be the card art (usually larger images with character names)
                const cardImg = imgs.find(img => 
                    img.src.includes('/supports/') || 
                    img.src.includes('/cards/') ||
                    (img.width > 100 && img.height > 100 && img.src.includes('umamusume'))
                );
                
                // Also look for images in the infobox
                const infoboxImg = document.querySelector('[class*="infobox"] img');
                
                return cardImg ? cardImg.src : (infoboxImg ? infoboxImg.src : null);
            }
        """)
        
        if img_url:
            # Clean filename - use stable ID from URL instead of card_id
            safe_name = re.sub(r'[<>:"/\\|?*]', '_', card_name)
            if stable_id:
                file_path = os.path.join(IMAGES_PATH, f"{stable_id}_{safe_name}.png")
            else:
                # Fallback to name-only if no stable ID (shouldn't happen)
                file_path = os.path.join(IMAGES_PATH, f"{safe_name}.png")
            
            # Skip if already exists
            if os.path.exists(file_path):
                # print(f"  Art already exists, skipping download")
                return file_path
                
            # Download image
            response = requests.get(img_url, timeout=10)
            if response.status_code == 200:
                with open(file_path, 'wb') as f:
                    f.write(response.content)
                return file_path
    except Exception as e:
        print(f"    Warning: Could not download image: {e}")
    
    return None

def scrape_support_card(page, url, conn, max_retries=3):
    """Scrape a single support card with key levels and retries"""
    
    for attempt in range(max_retries):
        try:
            page.goto(url, timeout=60000)
            page.wait_for_load_state("networkidle")
            page.wait_for_timeout(2000)  # Extra wait for JS rendering
            
            # Extract basic card info including type
            card_data = page.evaluate("""
                () => {
                    const h1 = document.querySelector('h1');
                    const title = h1 ? h1.textContent.trim() : '';
                    
                    // Find rarity image
                    const imgs = Array.from(document.querySelectorAll('img'));
                    const rarityImg = imgs.find(i => i.src.includes('rarity'));
                    
                    // Find type image - uses utx_ico_obtain pattern
                    const typeImg = imgs.find(i => i.src.includes('obtain_0'));
                    
                    return {
                        title: title,
                        rarityImgSrc: rarityImg ? rarityImg.src : null,
                        typeImgSrc: typeImg ? typeImg.src : null
                    };
                }
            """)
            
            # Parse name from title (remove rarity and "Support Card")
            full_title = card_data['title']
            name = re.sub(r'\s*\(SSR\)|\s*\(SR\)|\s*\(R\)', '', full_title)
            name = name.replace('Support Card', '').strip()
            
            if not name:
                print(f"  Warning: Empty name, skipping")
                return False
            
            rarity = parse_rarity_from_image(card_data['rarityImgSrc'])
            card_type = parse_type_from_image(card_data['typeImgSrc'])
            max_level = get_max_level_for_rarity(rarity)
            
            print(f"Scraping: {name} | {rarity} | {card_type} | Max Level: {max_level}")
            
            cur = conn.cursor()
            
            # Insert card using OR IGNORE to keep the same card_id if it exists
            cur.execute("""
                INSERT OR IGNORE INTO support_cards (name, rarity, card_type, max_level, gametora_url)
                VALUES (?, ?, ?, ?, ?)
            """, (name, rarity, card_type, max_level, url))
            
            # Update existing card to ensure data is fresh (without changing ID)
            cur.execute("""
                UPDATE support_cards 
                SET name = ?, rarity = ?, card_type = ?, max_level = ?
                WHERE gametora_url = ?
            """, (name, rarity, card_type, max_level, url))
            conn.commit()
            
            cur.execute("SELECT card_id FROM support_cards WHERE gametora_url = ?", (url,))
            card_id = cur.fetchone()[0]
            
            # Extract stable ID from URL for image filename
            stable_id = extract_stable_id_from_url(url)
            
            # Download character art using stable ID
            image_path = download_card_image(page, stable_id, name)
            if image_path:
                cur.execute("UPDATE support_cards SET image_path = ? WHERE card_id = ?", (image_path, card_id))
                conn.commit()
            
            # Clear existing effects for this card (in case of re-scrape)
            cur.execute("DELETE FROM support_effects WHERE card_id = ?", (card_id,))
            cur.execute("DELETE FROM support_hints WHERE card_id = ?", (card_id,))
            cur.execute("DELETE FROM event_skills WHERE event_id IN (SELECT event_id FROM support_events WHERE card_id = ?)", (card_id,))
            cur.execute("DELETE FROM support_events WHERE card_id = ?", (card_id,))
            
            # Scrape effects at key levels only
            scrape_effects_key_levels(page, card_id, max_level, cur)
            
            # Scrape hints
            scrape_hints(page, card_id, cur)
            
            # Scrape events
            scrape_events(page, card_id, cur)
            
            conn.commit()
            return True
            
        except Exception as e:
            print(f"  Attempt {attempt + 1} failed: {e}")
            if attempt < max_retries - 1:
                print(f"  Retrying...")
                time.sleep(1)
            else:
                print(f"  All retries failed for {url}")
                return False

def set_level(page, target_level):
    """Set the page to a specific level using JavaScript to click buttons"""
    
    # Use JavaScript to handle all the clicking - need to check children.length === 0 
    # and wait for level text to change after each click
    actual_level = page.evaluate("""
        async (targetLevel) => {
            // Get current level from the page
            const getLevel = () => {
                const el = Array.from(document.querySelectorAll('div')).find(d => 
                    d.textContent.trim().startsWith('Level ') && d.children.length === 0
                );
                if (!el) {
                    const text = document.body.innerText;
                    const match = text.match(/Level\\s*(\\d+)/i);
                    return match ? parseInt(match[1]) : 30;
                }
                return parseInt(el.textContent.replace('Level ', '').trim());
            };
            
            // Find a button by its exact text - MUST check children.length === 0
            const clickButton = (text) => {
                const btns = Array.from(document.querySelectorAll('div'));
                const btn = btns.find(d => d.textContent.trim() === text && d.children.length === 0);
                if (btn) {
                    btn.click();
                    return true;
                }
                return false;
            };
            
            let currentLevel = getLevel();
            
            // Navigate to target level
            while (currentLevel !== targetLevel) {
                let btnText;
                if (currentLevel > targetLevel) {
                    const diff = currentLevel - targetLevel;
                    btnText = diff >= 5 ? '-5' : '-1';
                } else {
                    const diff = targetLevel - currentLevel;
                    btnText = diff >= 5 ? '+5' : '+1';
                }
                
                if (!clickButton(btnText)) {
                    // Button not found, we might be at max/min level
                    break;
                }
                
                // CRITICAL: Wait for level text to actually change
                const startLevel = currentLevel;
                let start = Date.now();
                while (Date.now() - start < 1000) {
                    await new Promise(r => setTimeout(r, 50));
                    const newLevel = getLevel();
                    if (newLevel !== startLevel) {
                        currentLevel = newLevel;
                        break;
                    }
                }
                
                // If we're stuck, break out
                if (currentLevel === startLevel) {
                    break;
                }
            }
            
            // Final wait for effects to update
            await new Promise(r => setTimeout(r, 200));
            return getLevel();
        }
    """, target_level)
    
    return actual_level

def extract_effects(page):
    """Extract effects from current page state using proper DOM selectors"""
    effects = page.evaluate("""
        () => {
            const effects = [];
            
            // Method 1: Try to find effects using the specific class structure
            const effectContainers = document.querySelectorAll('[class*="effect__"]');
            
            effectContainers.forEach(container => {
                const text = container.innerText.trim();
                if (text.includes('Unlocked at level')) return;
                
                const fullText = text.split('\\n').join(' ');
                
                const patterns = [

                    // Basic Stats
                    { regex: /Speed Bonus\\s*(\\d+)/, name: 'Speed Bonus' },
                    { regex: /Stamina Bonus\\s*(\\d+)/, name: 'Stamina Bonus' },
                    { regex: /Power Bonus\\s*(\\d+)/, name: 'Power Bonus' },
                    { regex: /Guts Bonus\\s*(\\d+)/, name: 'Guts Bonus' },
                    { regex: /Wisdom Bonus\\s*(\\d+)/, name: 'Wisdom Bonus' },
                    { regex: /Wit Bonus\\s*(\\d+)/, name: 'Wisdom Bonus' }, // Alias Wit -> Wisdom
                    { regex: /Skill Pts Bonus\\s*(\\d+)/, name: 'Skill Pts Bonus' },
                    
                    // Initial Stats
                    { regex: /Initial Speed\\s*(\\d+)/, name: 'Initial Speed' },
                    { regex: /Initial Stamina\\s*(\\d+)/, name: 'Initial Stamina' },
                    { regex: /Initial Power\\s*(\\d+)/, name: 'Initial Power' },
                    { regex: /Initial Guts\\s*(\\d+)/, name: 'Initial Guts' },
                    { regex: /Initial Wisdom\\s*(\\d+)/, name: 'Initial Wisdom' },
                    { regex: /Initial Wit\\s*(\\d+)/, name: 'Initial Wisdom' }, // Alias Wit -> Wisdom

                    // Special Bonuses
                    { regex: /Friendship Bonus\\s*(\\d+%?)/, name: 'Friendship Bonus' },
                    { regex: /Mood Effect\\s*(\\d+%?)/, name: 'Mood Effect' },
                    { regex: /Motivation Effect\\s*(\\d+%?)/, name: 'Motivation Effect' },
                    { regex: /Training Effectiveness\\s*(\\d+%?)/, name: 'Training Effectiveness' },
                    { regex: /Race Bonus\\s*(\\d+%?)/, name: 'Race Bonus' },
                    { regex: /Fan Bonus\\s*(\\d+%?)/, name: 'Fan Bonus' },
                    
                    // Hints
                    { regex: /Hint Rate\\s*(\\d+%?)/, name: 'Hint Rate' },
                    { regex: /Hint Frequency\\s*(\\d+%?)/, name: 'Hint Rate' }, // Alias Frequency -> Rate
                    { regex: /Hint Lv Up\\s*(\\d+%?)/, name: 'Hint Lv Up' },
                    { regex: /Hint Levels\\s*Lv\\s*(\\d+)/, name: 'Hint Lv Up' }, // Alias Hint Levels -> Hint Lv Up
                    
                    // Specialty/Bond
                    { regex: /Starting Bond\\s*(\\d+)/, name: 'Starting Bond' },
                    { regex: /Initial Friendship Gauge\\s*(\\d+)/, name: 'Starting Bond' }, // Alias -> Starting Bond
                    { regex: /Specialty Rate\\s*(\\d+%?)/, name: 'Specialty Rate' },
                    { regex: /Specialty Priority\\s*(\\d+)/, name: 'Specialty Rate' }, // Alias Priority -> Rate (usually same concept)
                    
                    // Recovery/Usage
                    { regex: /Race Status\\s*(\\d+)/, name: 'Race Status' },
                    { regex: /Energy Discount\\s*(\\d+%?)/, name: 'Energy Discount' },
                    { regex: /Wit Friendship Recovery\\s*(\\d+)/, name: 'Wisdom Friendship Recovery' },
                    
                    // Catch-all Unique
                    { regex: /Unique Effect\\s*(.*)/, name: 'Unique Effect' },
                ];

                
                for (const p of patterns) {
                    const match = fullText.match(p.regex);
                    if (match && !effects.some(e => e.name === p.name)) {
                        effects.push({ name: p.name, value: match[1] });
                    }
                }
            });
            
            // Method 2: Fallback - scan entire page text
            if (effects.length === 0) {
                const bodyText = document.body.innerText;
                const lines = bodyText.split('\\n');
                
                const simplePatterns = [
                    /^(Friendship Bonus)\\s*(\\d+%?)$/,
                    /^(Mood Effect)\\s*(\\d+%?)$/,
                    /^(Race Bonus)\\s*(\\d+%?)$/,
                    /^(Fan Bonus)\\s*(\\d+%?)$/,
                    /^(Training Effectiveness)\\s*(\\d+%?)$/,
                ];
                
                for (const line of lines) {
                    const trimmed = line.trim();
                    for (const pattern of simplePatterns) {
                        const match = trimmed.match(pattern);
                        if (match && !effects.some(e => e.name === match[1])) {
                            effects.push({ name: match[1], value: match[2] });
                        }
                    }
                }
            }
            
            return effects;
        }
    """)
    return effects

def scrape_effects_key_levels(page, card_id, max_level, cur):
    """Scrape effects at key levels based on card rarity/max level"""
    
    # Determine which levels to scrape based on max_level (rarity)
    if max_level == 50: # SSR
        levels_to_scrape = [30, 35, 40, 45, 50]
    elif max_level == 45: # SR
        levels_to_scrape = [25, 30, 35, 40, 45]
    else: # R (max 40)
        levels_to_scrape = [20, 25, 30, 35, 40]
    
    # Filter out any that might exceed max somehow (safety check)
    levels_to_scrape = [l for l in levels_to_scrape if l <= max_level]
    
    for level in levels_to_scrape:
        actual_level = set_level(page, level)
        effects = extract_effects(page)
        
        for effect in effects:
            cur.execute("""
                INSERT INTO support_effects (card_id, level, effect_name, effect_value)
                VALUES (?, ?, ?, ?)
            """, (card_id, actual_level, effect['name'], effect['value']))
        
        print(f"  Level {actual_level}: {len(effects)} effects")

def scrape_hints(page, card_id, cur):
    """Scrape support hints/training skills"""
    hints = page.evaluate("""
        () => {
            const hints = [];
            const text = document.body.innerText;
            
            const hintsMatch = text.match(/Support [Hh]ints([\\s\\S]*?)(?:Training [Ee]vents|Skills from [Ee]vents|$)/);
            if (!hintsMatch) return hints;
            
            const hintsSection = hintsMatch[1];
            const lines = hintsSection.split('\\n');
            
            for (const line of lines) {
                const trimmed = line.trim();
                if (trimmed.length > 3 && trimmed.length < 60 && 
                    !trimmed.includes('Lv') && !trimmed.includes('%') &&
                    !trimmed.includes('Details') && trimmed[0] === trimmed[0].toUpperCase()) {
                    hints.push({ name: trimmed, description: '' });
                }
            }
            
            return hints.slice(0, 10);
        }
    """)
    
    for hint in hints:
        cur.execute("""
            INSERT INTO support_hints (card_id, hint_name, hint_description)
            VALUES (?, ?, ?)
        """, (card_id, hint.get('name', ''), hint.get('description', '')))
    
    if hints:
        print(f"  Found {len(hints)} hints")

def scrape_events(page, card_id, cur):
    """Scrape all events including Chain, Dates, Random and Special"""
    
    # Use a flag to avoid adding multiple console listeners
    if not hasattr(page, "_console_attached"):
        page.on("console", lambda msg: print(f"  [JS Console] {msg.text}") if "scrapping" not in msg.text.lower() else None)
        page._console_attached = True
    
    # 1. Build a map of skills from the 'Skills from events' summary section
    # This remains useful for identifying golden skills.
    skill_rarity_map = page.evaluate("""
        () => {
            const map = {};
            const containers = Array.from(document.querySelectorAll('div')).filter(d => 
                (d.innerText.includes('Details') || d.innerText.includes('Reward')) && d.innerText.length < 500
            );
            
            containers.forEach(c => {
                const nameNode = c.querySelector('b, span[font-weight="bold"], div[font-weight="bold"]');
                let name = nameNode ? nameNode.innerText.trim() : c.innerText.split('Details')[0].replace(/\\n/g, ' ').trim();
                
                if (name && name.length > 2) {
                    const style = window.getComputedStyle(c);
                    const nameStyle = nameNode ? window.getComputedStyle(nameNode) : null;
                    const isGold = style.backgroundImage.includes('linear-gradient') || 
                                   style.backgroundColor.includes('rgb(255, 193, 7)') ||
                                   (nameStyle && nameStyle.color === 'rgb(255, 193, 7)') ||
                                   c.className.includes('kkspcu');
                    
                    const normalized = name.toLowerCase().replace(/\\s+/g, ' ').replace(/[()（）-]/g, '').trim();
                    map[normalized] = isGold;
                }
            });
            return map;
        }
    """)
    
    # 2. Scrape all event types
    print("  Scraping all event categories...")
    all_events_data = page.evaluate("""
        async () => {
            const results = [];
            
            // Define categories to look for
            const categories = [
                { label: 'Chain Events', type: 'Chain' },
                { label: 'Dates', type: 'Date' },
                { label: 'Random Events', type: 'Random' },
                { label: 'Special Events', type: 'Special' }
            ];
            
            for (const cat of categories) {
                // Find category headers
                const headers = Array.from(document.querySelectorAll('div, span, h2, h3, h4')).filter(el => 
                    el.innerText.trim() === cat.label && el.children.length === 0
                );
                
                for (const header of headers) {
                    // Find buttons in the following siblings or parent siblings
                    let container = header.parentElement;
                    let foundButtons = [];
                    let attempts = 0;
                    
                    while (container && foundButtons.length === 0 && attempts < 5) {
                        foundButtons = Array.from(container.querySelectorAll('button')).filter(btn => {
                            const style = window.getComputedStyle(btn);
                            return style.display !== 'none' && style.visibility !== 'hidden';
                        });
                        
                        if (foundButtons.length === 0) {
                            // Check next siblings of the header's ancestors
                            let sibling = header;
                            let parent = header.parentElement;
                            while(parent && parent.tagName !== 'BODY') {
                                if (parent.innerText.includes(cat.label)) {
                                     const next = parent.nextElementSibling;
                                     if (next) {
                                         const nextBtns = Array.from(next.querySelectorAll('button'));
                                         if (nextBtns.length > 0) {
                                             foundButtons = nextBtns;
                                             break;
                                         }
                                     }
                                }
                                parent = parent.parentElement;
                            }
                        }
                        container = container.parentElement;
                        attempts++;
                    }
                    
                    // Scrape each button in the category
                    for (const btn of foundButtons) {
                        const eventName = btn.innerText.trim();
                        if (!eventName || results.some(r => r.name === eventName)) continue;
                        
                        // Count arrows for chain/date importance
                        const arrows = (eventName.match(/>|❯/g) || []).length;
                        
                        try {
                            btn.scrollIntoViewIfNeeded ? btn.scrollIntoViewIfNeeded() : null;
                            await new Promise(r => setTimeout(r, 100));
                            btn.click();
                            await new Promise(r => setTimeout(r, 500));
                            
                            const popovers = Array.from(document.querySelectorAll('div')).filter(d => 
                                d.innerText.includes(eventName) && 
                                window.getComputedStyle(d).zIndex > 50 &&
                                d.innerText.length < 2500
                            );
                            
                            if (popovers.length > 0) {
                                const pop = popovers[popovers.length - 1];
                                const hasOrDivider = pop.querySelector('[class*="divider_or"]') !== null || 
                                                     pop.innerText.includes('Randomly either') ||
                                                     pop.innerText.toLowerCase().includes(' or ');
                                                     
                                const skillLinks = Array.from(pop.querySelectorAll('span, a')).filter(el => 
                                    el.innerText.length > 2 && 
                                    !el.innerText.includes('Energy') && 
                                    (window.getComputedStyle(el).color === 'rgb(102, 107, 255)' || 
                                     el.className.includes('linkcolor'))
                                );
                                
                                const skills = [];
                                skillLinks.forEach(link => {
                                    const sName = link.innerText.trim();
                                    if (sName && !skills.some(s => s.name === sName)) {
                                        skills.push({ name: sName, is_or: hasOrDivider });
                                    }
                                });
                                
                                results.push({ 
                                    name: eventName, 
                                    type: cat.type, 
                                    skills: skills,
                                    arrows: arrows 
                                });
                            }
                            
                            document.body.click();
                            await new Promise(r => setTimeout(r, 100));
                        } catch (err) {
                            console.log(`Failed to scrape event ${eventName}: ${err.message}`);
                        }
                    }
                }
            }
            return results;
        }
    """)
    
    # 3. Store all found events and identify golden skills
    if all_events_data:
        # Determine max arrows for Chain and Dates to identify final step
        max_arrows = {
            'Chain': max([e['arrows'] for e in all_events_data if e['type'] == 'Chain'] + [0]),
            'Date': max([e['arrows'] for e in all_events_data if e['type'] == 'Date'] + [0])
        }
        
        for event in all_events_data:
            cur.execute("INSERT INTO support_events (card_id, event_name, event_type) VALUES (?, ?, ?)", 
                       (card_id, event['name'], event['type']))
            event_id = cur.lastrowid
            
            for skill in event['skills']:
                def normalize(s):
                    # Remove hint suffix and special characters
                    s = s.lower().split(' hint +')[0]
                    return re.sub(r'[()（）\-\s\+]', '', s).strip()
                
                n_name = normalize(skill['name'])
                is_gold = 0
                for k, gold in skill_rarity_map.items():
                    if normalize(k) == n_name:
                        is_gold = 1 if gold else 0
                        break
                
                # Heuristic: If it's the last step of a Chain or Date, it's likely gold
                if not is_gold and event['type'] in ['Chain', 'Date']:
                    if event['arrows'] >= 3 and event['arrows'] == max_arrows[event['type']]:
                        if len(event['skills']) <= 2 or "hint +" in skill['name'].lower():
                            is_gold = 1
                            print(f"  ✨ Heuristic Gold: {skill['name']} in {event['name']}")
                
                if is_gold: print(f"  ✨ Verified Gold: {skill['name']}")
                
                cur.execute("""
                    INSERT INTO event_skills (event_id, skill_name, is_gold, is_or)
                    VALUES (?, ?, ?, ?)
                """, (event_id, skill['name'], is_gold, 1 if skill['is_or'] else 0))
                
        print(f"  Scraped {len(all_events_data)} total events.")
    else:
        print(f"  No events found.")

def run_scraper():
    """ Run the web scraper to fetch card data from GameTora.com """
    print("=" * 60)
    print("GameTora Umamusume Support Card Scraper")
    print(f"Scraping effects at levels: {KEY_LEVELS}")
    print("=" * 60)
    
    # Initialize fresh database
    print("\nInitializing database...")
    init_database()
    
    # Create images directory
    os.makedirs(IMAGES_PATH, exist_ok=True)
    
    conn = get_conn()
    
    with sync_playwright() as p:
        print("\nLaunching browser...")
        browser = p.chromium.launch(headless=True)
        context = browser.new_context()
        page = context.new_page()
        
        # Get all card links
        links = scrape_all_support_links(page)
        
        print(f"\nStarting to scrape {len(links)} cards...")
        print("Including character art download.")
        print("This will take approximately 90-120 minutes.\n")
        
        success_count = 0
        fail_count = 0
        
        for i, url in enumerate(links, 1):
            print(f"\n[{i}/{len(links)}] ", end="")
            if scrape_support_card(page, url, conn):
                success_count += 1
            else:
                fail_count += 1
            
            time.sleep(0.3)
            
            if i % 50 == 0:
                print(f"\n{'='*40}")
                print(f"Progress: {i}/{len(links)} ({i*100//len(links)}%)")
                print(f"Success: {success_count}, Failed: {fail_count}")
                print(f"{'='*40}")
        
        browser.close()
    
    conn.close()
    
    print("\n" + "=" * 60)
    print("Scraping Complete!")
    print(f"Successfully scraped: {success_count} cards")
    print(f"Failed: {fail_count} cards")
    print(f"Images saved to: {IMAGES_PATH}")
    print("=" * 60)

if __name__ == "__main__":
    run_scraper()