refactor: Move maintenance scripts to a dedicated maintenance_scripts/ directory.
This commit is contained in:
23
maintenance_scripts/check_correlation.py
Normal file
23
maintenance_scripts/check_correlation.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def check_correlation():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT card_id, name FROM support_cards ORDER BY card_id ASC LIMIT 1")
|
||||
first_card = cur.fetchone()
|
||||
print(f"First card: {first_card}")
|
||||
|
||||
if first_card:
|
||||
# Check what events ID 1 points to
|
||||
cur.execute("SELECT event_name FROM support_events WHERE card_id = 1 LIMIT 1")
|
||||
first_event = cur.fetchone()
|
||||
print(f"First event for card_id 1: {first_event}")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_correlation()
|
||||
31
maintenance_scripts/check_counts.py
Normal file
31
maintenance_scripts/check_counts.py
Normal file
@@ -0,0 +1,31 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def check_counts():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM support_cards")
|
||||
print(f"support_cards: {cur.fetchone()[0]}")
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM support_events")
|
||||
print(f"support_events: {cur.fetchone()[0]}")
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM event_skills")
|
||||
print(f"event_skills: {cur.fetchone()[0]}")
|
||||
|
||||
# Check sample card_id from events
|
||||
cur.execute("SELECT card_id FROM support_events LIMIT 1")
|
||||
sample_id = cur.fetchone()
|
||||
if sample_id:
|
||||
print(f"Sample card_id from events: {sample_id[0]}")
|
||||
cur.execute("SELECT name FROM support_cards WHERE card_id = ?", (sample_id[0],))
|
||||
card_name = cur.fetchone()
|
||||
print(f"Matching card name: {card_name}")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_counts()
|
||||
33
maintenance_scripts/check_db.py
Normal file
33
maintenance_scripts/check_db.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "database", "umamusume.db")
|
||||
|
||||
def check_schema():
|
||||
if not os.path.exists(DB_PATH):
|
||||
print(f"Database not found at {DB_PATH}")
|
||||
return
|
||||
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
try:
|
||||
cur.execute("PRAGMA table_info(event_skills)")
|
||||
columns = [row[1] for row in cur.fetchall()]
|
||||
print(f"Columns in event_skills: {columns}")
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM event_skills")
|
||||
count = cur.fetchone()[0]
|
||||
print(f"Total skills in event_skills: {count}")
|
||||
|
||||
cur.execute("SELECT DISTINCT skill_name FROM event_skills WHERE is_gold = 1 LIMIT 5")
|
||||
gold_skills = cur.fetchall()
|
||||
print(f"Golden skills samples: {gold_skills}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_schema()
|
||||
16
maintenance_scripts/check_effects_ids.py
Normal file
16
maintenance_scripts/check_effects_ids.py
Normal file
@@ -0,0 +1,16 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def check_effects_card_ids():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT DISTINCT card_id FROM support_effects LIMIT 10")
|
||||
print(f"Distinct card_ids in support_effects: {[row[0] for row in cur.fetchall()]}")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_effects_card_ids()
|
||||
20
maintenance_scripts/check_event_ids.py
Normal file
20
maintenance_scripts/check_event_ids.py
Normal file
@@ -0,0 +1,20 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def check_event_card_ids():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT DISTINCT card_id FROM support_events LIMIT 20")
|
||||
ids = [row[0] for row in cur.fetchall()]
|
||||
print(f"Distinct card_ids in support_events: {ids}")
|
||||
|
||||
cur.execute("SELECT card_id, name FROM support_cards WHERE card_id IN (1, 2, 3, 4, 5)")
|
||||
print(f"Cards with IDs 1-5: {cur.fetchall()}")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_event_card_ids()
|
||||
21
maintenance_scripts/check_id_range.py
Normal file
21
maintenance_scripts/check_id_range.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def check_id_range():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT MIN(card_id), MAX(card_id) FROM support_cards")
|
||||
min_id, max_id = cur.fetchone()
|
||||
print(f"support_cards card_id range: {min_id} to {max_id}")
|
||||
|
||||
cur.execute("SELECT MIN(card_id), MAX(card_id) FROM support_events")
|
||||
min_ev_id, max_ev_id = cur.fetchone()
|
||||
print(f"support_events card_id range: {min_ev_id} to {max_ev_id}")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_id_range()
|
||||
20
maintenance_scripts/check_local_db.py
Normal file
20
maintenance_scripts/check_local_db.py
Normal file
@@ -0,0 +1,20 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
# Project root database
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def check_local_db():
|
||||
if not os.path.exists(DB_PATH):
|
||||
print("Local DB not found")
|
||||
return
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT MIN(card_id), MAX(card_id) FROM support_cards")
|
||||
print(f"Local support_cards range: {cur.fetchone()}")
|
||||
cur.execute("SELECT MIN(card_id), MAX(card_id) FROM support_events")
|
||||
print(f"Local support_events range: {cur.fetchone()}")
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_local_db()
|
||||
24
maintenance_scripts/check_offset.py
Normal file
24
maintenance_scripts/check_offset.py
Normal file
@@ -0,0 +1,24 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def check_offset():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Get first 5 SSR cards
|
||||
cur.execute("SELECT card_id, name, gametora_url FROM support_cards ORDER BY card_id ASC LIMIT 5")
|
||||
cards = cur.fetchall()
|
||||
print(f"Cards: {cards}")
|
||||
|
||||
# Check if there are events referring to IDs 1, 2, 3...
|
||||
for i in range(1, 6):
|
||||
cur.execute("SELECT event_name FROM support_events WHERE card_id = ? LIMIT 1", (i,))
|
||||
ev = cur.fetchone()
|
||||
print(f"ID {i} events: {ev}")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_offset()
|
||||
33
maintenance_scripts/check_orphans.py
Normal file
33
maintenance_scripts/check_orphans.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def check_orphans():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Check if event_id exists in support_events
|
||||
cur.execute("""
|
||||
SELECT COUNT(*)
|
||||
FROM event_skills es
|
||||
LEFT JOIN support_events se ON es.event_id = se.event_id
|
||||
WHERE se.event_id IS NULL
|
||||
""")
|
||||
orphans = cur.fetchone()[0]
|
||||
print(f"Orphaned skills (no matching event): {orphans}")
|
||||
|
||||
# Check if card_id exists in support_cards
|
||||
cur.execute("""
|
||||
SELECT COUNT(*)
|
||||
FROM support_events se
|
||||
LEFT JOIN support_cards sc ON se.card_id = sc.card_id
|
||||
WHERE sc.card_id IS NULL
|
||||
""")
|
||||
orphaned_events = cur.fetchone()[0]
|
||||
print(f"Orphaned events (no matching card): {orphaned_events}")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_orphans()
|
||||
20
maintenance_scripts/check_paths.py
Normal file
20
maintenance_scripts/check_paths.py
Normal file
@@ -0,0 +1,20 @@
|
||||
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = r"y:\Keith\umamusuma card application\database\umamusume.db"
|
||||
print(f"Checking DB at: {DB_PATH}")
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT card_id, name, image_path FROM support_cards LIMIT 5")
|
||||
rows = cur.fetchall()
|
||||
|
||||
print("\nSample Card Data:")
|
||||
for row in rows:
|
||||
print(f"ID: {row[0]}, Name: {row[1]}, Path: {row[2]}")
|
||||
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
19
maintenance_scripts/check_seed_db.py
Normal file
19
maintenance_scripts/check_seed_db.py
Normal file
@@ -0,0 +1,19 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume_seed.db")
|
||||
|
||||
def check_seed_db():
|
||||
if not os.path.exists(DB_PATH):
|
||||
print("Seed DB not found")
|
||||
return
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT MIN(card_id), MAX(card_id) FROM support_cards")
|
||||
print(f"Seed support_cards range: {cur.fetchone()}")
|
||||
cur.execute("SELECT MIN(card_id), MAX(card_id) FROM support_events")
|
||||
print(f"Seed support_events range: {cur.fetchone()}")
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_seed_db()
|
||||
42
maintenance_scripts/debug_db.py
Normal file
42
maintenance_scripts/debug_db.py
Normal file
@@ -0,0 +1,42 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
def debug_db():
|
||||
db_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "database", "umamusume.db")
|
||||
if not os.path.exists(db_path):
|
||||
print(f"Database not found at {db_path}")
|
||||
return
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
|
||||
print("--- Database Debug ---")
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM support_cards")
|
||||
print(f"Total support cards: {cur.fetchone()[0]}")
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM owned_cards")
|
||||
owned_count = cur.fetchone()[0]
|
||||
print(f"Owned cards count: {owned_count}")
|
||||
|
||||
cur.execute("""
|
||||
SELECT oc.card_id, sc.name
|
||||
FROM owned_cards oc
|
||||
LEFT JOIN support_cards sc ON oc.card_id = sc.card_id
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
|
||||
print("\nOwned cards details:")
|
||||
for card_id, name in rows:
|
||||
print(f" ID: {card_id}, Name: {name}")
|
||||
|
||||
orphaned = [row[0] for row in rows if row[1] is None]
|
||||
if orphaned:
|
||||
print(f"\nFound {len(orphaned)} orphaned owned cards (Card IDs: {orphaned})")
|
||||
else:
|
||||
print("\nNo orphaned owned cards found.")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
debug_db()
|
||||
118
maintenance_scripts/debug_kitasan_scrape.py
Normal file
118
maintenance_scripts/debug_kitasan_scrape.py
Normal file
@@ -0,0 +1,118 @@
|
||||
import os
|
||||
import sys
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
# Add parent dir to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
def debug_kitasan_scrape():
|
||||
url = "https://gametora.com/umamusume/supports/30028-kitasan-black"
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
page.goto(url)
|
||||
page.wait_for_load_state("networkidle")
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
# 1. Get Skill Rarity Map
|
||||
rarity_map = page.evaluate("""
|
||||
() => {
|
||||
const map = {};
|
||||
const sections = Array.from(document.querySelectorAll('div, span, h3')).filter(el =>
|
||||
el.innerText.trim().startsWith('Skills from events')
|
||||
);
|
||||
if (sections.length === 0) return { error: "Section not found" };
|
||||
|
||||
const root = sections[0].closest('div');
|
||||
const containers = Array.from(root.querySelectorAll('div')).filter(d =>
|
||||
d.innerText.includes('Details') && d.children.length > 1
|
||||
);
|
||||
|
||||
containers.forEach(c => {
|
||||
const textNodes = Array.from(c.querySelectorAll('div, span')).filter(n => n.children.length === 0);
|
||||
const name = textNodes[0] ? textNodes[0].innerText.trim() : "";
|
||||
|
||||
if (name && name.length > 1 && !name.includes('Details')) {
|
||||
const style = window.getComputedStyle(c);
|
||||
const isGold = style.backgroundImage.includes('linear-gradient') ||
|
||||
style.backgroundColor.includes('rgb(255, 193, 7)') ||
|
||||
c.className.includes('kkspcu');
|
||||
map[name] = isGold;
|
||||
}
|
||||
});
|
||||
return map;
|
||||
}
|
||||
""")
|
||||
print(f"Skill Rarity Map: {rarity_map}")
|
||||
|
||||
# 2. Click Golden Perk Button
|
||||
page.evaluate("() => { const h = Array.from(document.querySelectorAll('h2, h1')).find(el => el.innerText.includes('Training Events')); if (h) h.scrollIntoView(); }")
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
btn_found = page.evaluate("""
|
||||
() => {
|
||||
const labels = Array.from(document.querySelectorAll('div, span, h2, h3')).filter(el =>
|
||||
el.innerText.trim() === 'Chain Events'
|
||||
);
|
||||
const buttons = [];
|
||||
labels.forEach(label => {
|
||||
let container = label.parentElement;
|
||||
while (container && container.querySelectorAll('button').length === 0) {
|
||||
container = container.nextElementSibling || container.parentElement;
|
||||
if (container && container.tagName === 'BODY') break;
|
||||
}
|
||||
if (container) {
|
||||
const btns = Array.from(container.querySelectorAll('button'));
|
||||
btns.forEach(btn => {
|
||||
const text = btn.innerText.trim();
|
||||
if (text.includes('>') || text.includes('❯')) buttons.push(btn);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
let goldenBtn = buttons.find(b => b.innerText.includes('❯❯❯'));
|
||||
if (!goldenBtn) {
|
||||
// Fallback to max arrows
|
||||
let maxArrows = 0;
|
||||
buttons.forEach(b => {
|
||||
const count = (b.innerText.match(/>|❯/g) || []).length;
|
||||
if (count > maxArrows) { maxArrows = count; goldenBtn = b; }
|
||||
});
|
||||
}
|
||||
|
||||
if (goldenBtn) {
|
||||
goldenBtn.click();
|
||||
return goldenBtn.innerText;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
""")
|
||||
print(f"Clicked button: {btn_found}")
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
# 3. Get Skills from Tooltip
|
||||
tooltip_skills = page.evaluate("""
|
||||
() => {
|
||||
const popovers = Array.from(document.querySelectorAll('div')).filter(d =>
|
||||
window.getComputedStyle(d).zIndex > 50 &&
|
||||
d.innerText.length < 2500
|
||||
);
|
||||
if (popovers.length === 0) return { error: "No popovers found" };
|
||||
|
||||
const pop = popovers[popovers.length - 1];
|
||||
const skillLinks = Array.from(pop.querySelectorAll('span, a')).filter(el =>
|
||||
el.innerText.length > 2 &&
|
||||
!el.innerText.includes('Energy') &&
|
||||
!el.innerText.includes('bond')
|
||||
);
|
||||
return skillLinks.map(s => s.innerText.trim());
|
||||
}
|
||||
""")
|
||||
print(f"Tooltip Skills: {tooltip_skills}")
|
||||
|
||||
browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
debug_kitasan_scrape()
|
||||
87
maintenance_scripts/deep_repair.py
Normal file
87
maintenance_scripts/deep_repair.py
Normal file
@@ -0,0 +1,87 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Ensure we can import from the project
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from scraper.gametora_scraper import scrape_support_card, sync_playwright
|
||||
from db.db_queries import DB_PATH, repair_orphaned_data, cleanup_orphaned_data
|
||||
|
||||
def deep_repair():
|
||||
print("=" * 60)
|
||||
print("Umamusume Card Manager - Deep Database Repair")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. Run basic repair and cleanup
|
||||
print("\nStep 1: Cleaning up corrupted records...")
|
||||
repair_orphaned_data()
|
||||
cleanup_orphaned_data()
|
||||
|
||||
# 2. Identify missing data
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("""
|
||||
SELECT card_id, name, gametora_url
|
||||
FROM support_cards
|
||||
WHERE card_id NOT IN (SELECT DISTINCT card_id FROM support_events)
|
||||
AND rarity = 'SSR'
|
||||
""")
|
||||
ssr_missing = cur.fetchall()
|
||||
|
||||
cur.execute("""
|
||||
SELECT card_id, name, gametora_url
|
||||
FROM support_cards
|
||||
WHERE card_id NOT IN (SELECT DISTINCT card_id FROM support_events)
|
||||
AND rarity != 'SSR'
|
||||
""")
|
||||
others_missing = cur.fetchall()
|
||||
|
||||
total_missing = len(ssr_missing) + len(others_missing)
|
||||
if total_missing == 0:
|
||||
print("\n✅ No missing data detected. Your database is healthy!")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
print(f"\nDetected {total_missing} cards with missing event/skill data.")
|
||||
print(f"- SSR cards: {len(ssr_missing)}")
|
||||
print(f"- SR/R cards: {len(others_missing)}")
|
||||
|
||||
print("\nStep 2: Re-scraping missing data from GameTora...")
|
||||
print("This may take some time depending on your internet connection.")
|
||||
print("Press Ctrl+C to stop at any time.")
|
||||
|
||||
try:
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
# Prioritize SSRs
|
||||
to_process = ssr_missing + others_missing
|
||||
count = 0
|
||||
for card_id, name, url in to_process:
|
||||
count += 1
|
||||
percent = (count / total_missing) * 100
|
||||
print(f"[{count}/{total_missing} - {percent:.1f}%] Repairing: {name}")
|
||||
try:
|
||||
scrape_support_card(page, url, conn)
|
||||
except Exception as e:
|
||||
print(f" ❌ Error: {e}")
|
||||
|
||||
browser.close()
|
||||
except KeyboardInterrupt:
|
||||
print("\n⚠️ Repair interrupted by user.")
|
||||
except Exception as e:
|
||||
print(f"\n❌ A fatal error occurred during scrape: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Repair process finished.")
|
||||
print("You can now restart the application.")
|
||||
print("=" * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
deep_repair()
|
||||
58
maintenance_scripts/fast_rescrape.py
Normal file
58
maintenance_scripts/fast_rescrape.py
Normal file
@@ -0,0 +1,58 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add parent dir to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from scraper.gametora_scraper import scrape_support_card, sync_playwright
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def fast_rescrape():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Find cards that have NO events
|
||||
cur.execute("""
|
||||
SELECT card_id, name, gametora_url
|
||||
FROM support_cards
|
||||
WHERE card_id NOT IN (SELECT DISTINCT card_id FROM support_events)
|
||||
AND rarity = 'SSR'
|
||||
""")
|
||||
cards_to_rescrape = cur.fetchall()
|
||||
|
||||
print(f"Found {len(cards_to_rescrape)} SSR cards missing event data.")
|
||||
|
||||
if not cards_to_rescrape:
|
||||
conn.close()
|
||||
return
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
count = 0
|
||||
for card_id, name, url in cards_to_rescrape:
|
||||
count += 1
|
||||
print(f"[{count}/{len(cards_to_rescrape)}] Re-scraping: {name}")
|
||||
try:
|
||||
# We need to pass the same connection or use a different scraper function
|
||||
# The existing scrape_support_card re-inserts the card too.
|
||||
# Since we fixed the scraper to use INSERT OR IGNORE, it's safe!
|
||||
from scraper.gametora_scraper import scrape_support_card
|
||||
scrape_support_card(page, url, conn)
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
if count % 10 == 0:
|
||||
print("--- Progress Checkpoint ---")
|
||||
|
||||
browser.close()
|
||||
|
||||
conn.close()
|
||||
print("Fast re-scrape complete.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
fast_rescrape()
|
||||
53
maintenance_scripts/fast_rescrape_limited.py
Normal file
53
maintenance_scripts/fast_rescrape_limited.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add parent dir to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from scraper.gametora_scraper import scrape_support_card, sync_playwright
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def fast_rescrape_limited():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Find cards that have NO events
|
||||
cur.execute("""
|
||||
SELECT card_id, name, gametora_url
|
||||
FROM support_cards
|
||||
WHERE card_id NOT IN (SELECT DISTINCT card_id FROM support_events)
|
||||
ORDER BY rarity DESC, card_id ASC
|
||||
LIMIT 50
|
||||
""")
|
||||
cards_to_rescrape = cur.fetchall()
|
||||
|
||||
print(f"Found {len(cards_to_rescrape)} cards to re-scrape (Limited to 50).")
|
||||
|
||||
if not cards_to_rescrape:
|
||||
conn.close()
|
||||
return
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
count = 0
|
||||
for card_id, name, url in cards_to_rescrape:
|
||||
count += 1
|
||||
print(f"[{count}/{len(cards_to_rescrape)}] Re-scraping: {name}")
|
||||
try:
|
||||
from scraper.gametora_scraper import scrape_support_card
|
||||
scrape_support_card(page, url, conn)
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
browser.close()
|
||||
|
||||
conn.close()
|
||||
print("Limited re-scrape complete.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
fast_rescrape_limited()
|
||||
37
maintenance_scripts/repair_db.py
Normal file
37
maintenance_scripts/repair_db.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
DB_PATH = os.path.join("database", "umamusume.db")
|
||||
|
||||
def repair_db():
|
||||
if not os.path.exists(DB_PATH):
|
||||
return
|
||||
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cur = conn.cursor()
|
||||
|
||||
print("Repairing database...")
|
||||
|
||||
# 1. Remove all orphans
|
||||
cur.execute("DELETE FROM support_effects WHERE card_id NOT IN (SELECT card_id FROM support_cards)")
|
||||
print(f"Removed {cur.rowcount} orphaned effects")
|
||||
|
||||
cur.execute("DELETE FROM support_hints WHERE card_id NOT IN (SELECT card_id FROM support_cards)")
|
||||
print(f"Removed {cur.rowcount} orphaned hints")
|
||||
|
||||
cur.execute("DELETE FROM event_skills WHERE event_id NOT IN (SELECT event_id FROM support_events)")
|
||||
print(f"Removed {cur.rowcount} orphaned event skills")
|
||||
|
||||
cur.execute("DELETE FROM support_events WHERE card_id NOT IN (SELECT card_id FROM support_cards)")
|
||||
print(f"Removed {cur.rowcount} orphaned events")
|
||||
|
||||
# 2. Cleanup owned_cards and deck_slots
|
||||
cur.execute("DELETE FROM owned_cards WHERE card_id NOT IN (SELECT card_id FROM support_cards)")
|
||||
cur.execute("DELETE FROM deck_slots WHERE card_id NOT IN (SELECT card_id FROM support_cards)")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print("Repair complete.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
repair_db()
|
||||
62
maintenance_scripts/test_gold_scrape.py
Normal file
62
maintenance_scripts/test_gold_scrape.py
Normal file
@@ -0,0 +1,62 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add parent dir to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from scraper.gametora_scraper import scrape_support_card, sync_playwright
|
||||
from db.db_queries import get_conn
|
||||
|
||||
def test_golden_perk():
|
||||
print("Testing Golden Perk Scraping for Fine Motion...")
|
||||
url = "https://gametora.com/umamusume/supports/30010-fine-motion"
|
||||
|
||||
conn = get_conn()
|
||||
cur = conn.cursor()
|
||||
|
||||
# 1. Clean previous data for this specific card
|
||||
cur.execute("SELECT card_id FROM support_cards WHERE gametora_url = ?", (url,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
card_id = row[0]
|
||||
cur.execute("DELETE FROM event_skills WHERE event_id IN (SELECT event_id FROM support_events WHERE card_id = ?)", (card_id,))
|
||||
cur.execute("DELETE FROM support_events WHERE card_id = ?", (card_id,))
|
||||
conn.commit()
|
||||
|
||||
# 2. Scrape
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
success = scrape_support_card(page, url, conn)
|
||||
print(f"Scrape success: {success}")
|
||||
browser.close()
|
||||
|
||||
# 3. Verify results
|
||||
cur.execute("""
|
||||
SELECT se.event_name, es.skill_name, es.is_gold
|
||||
FROM support_events se
|
||||
JOIN event_skills es ON se.event_id = es.event_id
|
||||
JOIN support_cards sc ON se.card_id = sc.card_id
|
||||
WHERE sc.gametora_url = ?
|
||||
""", (url,))
|
||||
|
||||
skills = cur.fetchall()
|
||||
print(f"\nSkills found for Kitasan Black:")
|
||||
found_gold = False
|
||||
for event_name, skill_name, is_gold in skills:
|
||||
status = "✨ GOLD" if is_gold else "Normal"
|
||||
print(f"- [{status}] {event_name}: {skill_name}")
|
||||
if is_gold: found_gold = True
|
||||
|
||||
if found_gold:
|
||||
print("\n✅ SUCCESS: Golden Perk identified correctly!")
|
||||
else:
|
||||
print("\n❌ FAILURE: No golden perks found.")
|
||||
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_golden_perk()
|
||||
111
maintenance_scripts/test_scraper.py
Normal file
111
maintenance_scripts/test_scraper.py
Normal file
@@ -0,0 +1,111 @@
|
||||
|
||||
import sqlite3
|
||||
import os
|
||||
from playwright.sync_api import sync_playwright
|
||||
import sys
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
def test_scrape_events(url):
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
page = browser.new_page()
|
||||
print(f"Testing URL: {url}")
|
||||
page.goto(url)
|
||||
page.wait_for_timeout(2000)
|
||||
|
||||
# 1. First, build a map of skills from the 'Skills from events' summary section
|
||||
skill_rarity_map = page.evaluate("""
|
||||
() => {
|
||||
const map = {};
|
||||
const sections = Array.from(document.querySelectorAll('div')).filter(d => d.innerText.includes('Skills from events'));
|
||||
if (sections.length === 0) return { error: 'No Skills from events section' };
|
||||
|
||||
const containers = sections[0].parentElement.querySelectorAll('div[class*="sc-"]');
|
||||
containers.forEach(c => {
|
||||
const nameNode = c.querySelector('div[font-weight="bold"], span[font-weight="bold"], b');
|
||||
const name = nameNode ? nameNode.innerText.trim() : c.innerText.split('\\n')[0].trim();
|
||||
if (name && name.length > 2) {
|
||||
const isGold = c.className.includes('kkspcu') ||
|
||||
window.getComputedStyle(c).backgroundColor.includes('rgb(255, 193, 7)') ||
|
||||
c.innerText.includes('✨');
|
||||
map[name] = isGold;
|
||||
}
|
||||
});
|
||||
return map;
|
||||
}
|
||||
""")
|
||||
print(f"Skill Rarity Map: {skill_rarity_map}")
|
||||
|
||||
# 2. Scrape ONLY the LAST chain event (Golden Perk) with OR options
|
||||
golden_perk_data = page.evaluate("""
|
||||
async () => {
|
||||
const getChainEventButtons = () => {
|
||||
const buttons = [];
|
||||
const headers = Array.from(document.querySelectorAll('div, h2, h3, span')).filter(el =>
|
||||
el.innerText.includes('Chain Events')
|
||||
);
|
||||
|
||||
headers.forEach(header => {
|
||||
const container = header.parentElement;
|
||||
if (container) {
|
||||
const btns = Array.from(container.querySelectorAll('button'));
|
||||
btns.forEach(btn => {
|
||||
const text = btn.innerText.trim();
|
||||
const isVisible = btn.offsetWidth > 0;
|
||||
if (isVisible && text && text.includes('>') && !text.includes('Events')) {
|
||||
buttons.push(btn);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
return buttons;
|
||||
};
|
||||
|
||||
const buttons = getChainEventButtons();
|
||||
if (buttons.length === 0) return { error: 'No chain event buttons found' };
|
||||
|
||||
let goldenPerkButton = null;
|
||||
let maxArrows = 0;
|
||||
for (const btn of buttons) {
|
||||
const text = btn.innerText.trim();
|
||||
const arrowCount = (text.match(/>/g) || []).length;
|
||||
if (arrowCount > maxArrows) {
|
||||
maxArrows = arrowCount;
|
||||
goldenPerkButton = btn;
|
||||
}
|
||||
}
|
||||
|
||||
if (!goldenPerkButton) return { error: 'No golden perk button identified' };
|
||||
|
||||
const eventName = goldenPerkButton.innerText.trim();
|
||||
goldenPerkButton.click();
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
const popovers = Array.from(document.querySelectorAll('div')).filter(d =>
|
||||
d.innerText.includes(eventName) &&
|
||||
window.getComputedStyle(d).zIndex > 50
|
||||
);
|
||||
|
||||
if (popovers.length === 0) return { error: 'Popover not found', eventName: eventName };
|
||||
|
||||
const pop = popovers[popovers.length - 1];
|
||||
const skillLinks = Array.from(pop.querySelectorAll('span, a')).filter(el =>
|
||||
el.innerText.length > 2 &&
|
||||
(window.getComputedStyle(el).color === 'rgb(102, 107, 255)' ||
|
||||
el.className.includes('linkcolor'))
|
||||
);
|
||||
|
||||
return {
|
||||
name: eventName,
|
||||
skills: skillLinks.map(l => l.innerText.trim())
|
||||
};
|
||||
}
|
||||
""")
|
||||
print(f"Golden Perk Data: {golden_perk_data}")
|
||||
browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test with Gentildonna (verified URL from subagent)
|
||||
test_scrape_events("https://gametora.com/umamusume/supports/30186-gentildonna")
|
||||
Reference in New Issue
Block a user