518 lines
22 KiB
Python
518 lines
22 KiB
Python
"""
|
||
e-Gov ログイン Playwright スクレイパー
|
||
GビズID プレミアム (TOTP) によるログインを自動化します。
|
||
"""
|
||
import asyncio
|
||
import json
|
||
import base64
|
||
from pathlib import Path
|
||
|
||
from app.modules.egov.state import state, EGovStatus
|
||
|
||
# ── 定数 ─────────────────────────────────────────────────────
|
||
EGOV_TOP_URL = "https://shinsei.e-gov.go.jp/"
|
||
EGOV_DATA_DIR = Path("/app/egov_data")
|
||
COOKIES_PATH = EGOV_DATA_DIR / "cookies.json"
|
||
|
||
# e-Gov ログインリンク
|
||
SEL_LOGIN_LINK = 'a:has-text("ログイン"), button:has-text("ログイン")'
|
||
SEL_GBIZID_BTN = 'a:has-text("GビズID"), button:has-text("GビズID"), [alt*="GビズID"]'
|
||
|
||
# GビズID 認証フォーム（Keycloak 標準 DOM）
|
||
SEL_USERNAME = '#username, input[name="username"]'
|
||
SEL_PASSWORD = '#password, input[name="password"]'
|
||
SEL_SUBMIT = 'input[type="submit"], button[type="submit"]'
|
||
|
||
# TOTP フォーム（Keycloak 標準 DOM）
|
||
SEL_OTP = '#otp, input[name="otp"], #totp, input[name="totp"]'
|
||
|
||
# ログイン済み判定キーワード（「ログアウト」ボタンは認証済みページにのみ存在）
|
||
LOGGEDIN_MARKERS = ["ログアウト", "logout"]
|
||
|
||
# MFA 待機タイムアウト（秒）
|
||
MFA_TIMEOUT = 300
|
||
|
||
|
||
# ── メイン処理 ───────────────────────────────────────────────
|
||
|
||
async def run_login(user_id: str, password: str):
|
||
"""e-Gov ログイン処理（FastAPI BackgroundTask として実行）"""
|
||
try:
|
||
from playwright.async_api import async_playwright # 遅延import
|
||
|
||
# 既存ブラウザセッションを閉じてからリスタート
|
||
await state.close_browser()
|
||
|
||
state.status = EGovStatus.STARTING
|
||
state.message = "ブラウザを起動中..."
|
||
EGOV_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
# context manager ではなく .start() で起動 → 手動で lifetime を管理
|
||
pw = await async_playwright().start()
|
||
browser = await pw.chromium.launch(
|
||
headless=True,
|
||
args=["--no-sandbox", "--disable-dev-shm-usage"],
|
||
)
|
||
context = await _load_context(browser)
|
||
page = await context.new_page()
|
||
page.set_default_timeout(30_000)
|
||
|
||
# ── e-Gov トップにアクセス ──────────────────────
|
||
state.message = "e-Gov にアクセス中..."
|
||
await page.goto(EGOV_TOP_URL, wait_until="domcontentloaded")
|
||
|
||
await _screenshot(page)
|
||
|
||
# ── セッション復元チェック ──────────────────────
|
||
if await _is_logged_in(page):
|
||
await _save_cookies(context)
|
||
# ブラウザを閉じずに保持（download で再利用）
|
||
state._playwright = pw
|
||
state._browser = browser
|
||
state._context = context
|
||
state._page = page
|
||
state.status = EGovStatus.LOGGED_IN
|
||
state.message = "ログイン済み（セッション復元）"
|
||
return
|
||
|
||
# ── ログイン → GビズID へ誘導 ──────────────────
|
||
state.message = "GビズID ログインページへ移動中..."
|
||
await _navigate_to_gbizid(page)
|
||
await _screenshot(page)
|
||
|
||
# ── ナビゲーション後の状態を確認（SSO 自動ログインの検出）──────
|
||
state.message = "ログイン画面を確認中..."
|
||
await page.wait_for_timeout(2000)
|
||
await _screenshot(page)
|
||
|
||
# Keycloak SSO が有効で e-Gov に自動リダイレクトされた場合
|
||
if await _is_logged_in(page):
|
||
await _save_cookies(context)
|
||
state._playwright = pw
|
||
state._browser = browser
|
||
state._context = context
|
||
state._page = page
|
||
state.status = EGovStatus.LOGGED_IN
|
||
state.message = "ログイン完了（SSO自動ログイン）"
|
||
return
|
||
|
||
# ── ID / パスワード入力（フォームがある場合のみ）─────────────
|
||
state.message = "GビズID・パスワードを入力中..."
|
||
username_visible = False
|
||
try:
|
||
await page.wait_for_selector(SEL_USERNAME, timeout=12_000)
|
||
username_visible = True
|
||
except Exception:
|
||
pass # OTP フォームのみ表示される場合（記憶されたセッション）
|
||
|
||
if username_visible:
|
||
await page.fill(SEL_USERNAME, user_id)
|
||
await page.fill(SEL_PASSWORD, password)
|
||
await _screenshot(page)
|
||
await page.locator(SEL_SUBMIT).first.click()
|
||
else:
|
||
# OTP フォームが表示されているか確認
|
||
otp_showing = await page.evaluate(
|
||
"() => !!document.querySelector('#otp, input[name=\"otp\"], #totp, input[name=\"totp\"]')"
|
||
)
|
||
if not otp_showing:
|
||
current_url = page.url
|
||
await _screenshot(page)
|
||
# ブラウザを保持: /egov/debug-page でページ状態を診断できるよう state._page を設定
|
||
state._playwright = pw
|
||
state._browser = browser
|
||
state._context = context
|
||
state._page = page
|
||
state.status = EGovStatus.ERROR
|
||
state.message = (
|
||
f"GビズID ログインフォームが見つかりません。"
|
||
f"現在URL: {current_url} |"
|
||
f" /egov/debug-page で詳細を確認してください。"
|
||
)
|
||
return
|
||
# OTP フォームのみ → ID/PW ステップをスキップして次へ
|
||
|
||
# ── TOTP ページを待機 ───────────────────────────
|
||
state.message = "認証コード入力ページへ移動中..."
|
||
try:
|
||
await page.wait_for_selector(SEL_OTP, timeout=12_000)
|
||
await _screenshot(page)
|
||
except Exception:
|
||
await page.wait_for_load_state("domcontentloaded")
|
||
await _screenshot(page)
|
||
if await _is_logged_in(page):
|
||
await _save_cookies(context)
|
||
state._playwright = pw
|
||
state._browser = browser
|
||
state._context = context
|
||
state._page = page
|
||
state.status = EGovStatus.LOGGED_IN
|
||
state.message = "ログイン完了（MFA なし）"
|
||
return
|
||
await state.close_browser()
|
||
raise RuntimeError(
|
||
"認証コードページへの遷移に失敗しました。"
|
||
"GビズID またはパスワードをご確認ください。"
|
||
)
|
||
|
||
# ── ユーザーの MFA 入力を待つ ───────────────────
|
||
state.status = EGovStatus.WAITING_MFA
|
||
state.message = "GビズID アプリの認証コード（6桁）を入力してください"
|
||
|
||
try:
|
||
await asyncio.wait_for(state.mfa_event.wait(), timeout=MFA_TIMEOUT)
|
||
except asyncio.TimeoutError:
|
||
await state.close_browser()
|
||
state.status = EGovStatus.ERROR
|
||
state.message = f"タイムアウト: {MFA_TIMEOUT}秒以内に認証コードが入力されませんでした"
|
||
return
|
||
|
||
# ── MFA コードを送信 ────────────────────────────
|
||
mfa_code = state.mfa_code
|
||
state.message = "認証コードを送信中..."
|
||
await page.fill(SEL_OTP, mfa_code)
|
||
await page.locator(SEL_SUBMIT).first.click()
|
||
|
||
# ── ログイン完了を確認 ──────────────────────────
|
||
await page.wait_for_load_state("domcontentloaded", timeout=15_000)
|
||
try:
|
||
await page.wait_for_load_state("networkidle", timeout=10_000)
|
||
except Exception:
|
||
pass
|
||
await _screenshot(page)
|
||
|
||
if await _is_logged_in(page):
|
||
await _save_cookies(context)
|
||
# ブラウザを閉じずに保持
|
||
state._playwright = pw
|
||
state._browser = browser
|
||
state._context = context
|
||
state._page = page
|
||
state.status = EGovStatus.LOGGED_IN
|
||
state.message = "ログイン完了"
|
||
else:
|
||
await state.close_browser()
|
||
state.status = EGovStatus.ERROR
|
||
state.message = "ログインに失敗しました（認証コードが間違っている可能性があります）"
|
||
|
||
except Exception as exc:
|
||
await state.close_browser()
|
||
state.status = EGovStatus.ERROR
|
||
state.message = f"エラー: {exc}"
|
||
|
||
|
||
# ── 電子送達ダウンロード ──────────────────────────────────────
|
||
|
||
DOWNLOADS_DIR = EGOV_DATA_DIR / "downloads"
|
||
|
||
SEL_DOWNLOAD_BTN = 'button:has-text("通知ファイルをダウンロード"), a:has-text("通知ファイルをダウンロード")'
|
||
|
||
|
||
async def run_download():
|
||
"""電子送達一覧から最初のファイルをダウンロードする（BackgroundTask）"""
|
||
try:
|
||
if state.status != EGovStatus.LOGGED_IN:
|
||
state.status = EGovStatus.ERROR
|
||
state.message = "ログインしてからダウンロードを実行してください"
|
||
return
|
||
|
||
if state._page is None:
|
||
state.status = EGovStatus.ERROR
|
||
state.message = "ブラウザセッションがありません。再ログインしてください"
|
||
return
|
||
|
||
state.status = EGovStatus.DOWNLOADING
|
||
DOWNLOADS_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
page = state._page
|
||
|
||
# ─────────────────────────────────────────────────────
|
||
# 診断ステージ1: goto 前のページ状態（ログイン直後のページを確認）
|
||
# ─────────────────────────────────────────────────────
|
||
try:
|
||
pre = await page.evaluate("""
|
||
() => {
|
||
const b = document.body ? document.body.innerText : '';
|
||
return {
|
||
url: location.href,
|
||
hasLogout: b.includes('ログアウト'),
|
||
hasMyPage: b.includes('マイページ'),
|
||
hasDelivery: b.includes('電子送達'),
|
||
bodyLen: b.length,
|
||
};
|
||
}
|
||
""")
|
||
state.message = (
|
||
f"[診断1/goto前] url={pre['url']} "
|
||
f"ログアウト={pre['hasLogout']} マイページ={pre['hasMyPage']} "
|
||
f"電子送達={pre['hasDelivery']} bodyLen={pre['bodyLen']}"
|
||
)
|
||
except Exception as e:
|
||
state.message = f"[診断1エラー] {e}"
|
||
pre = {"hasLogout": False}
|
||
await _screenshot(page)
|
||
|
||
# ── 電子送達一覧へ移動 ──────────────────────────
|
||
await page.goto(
|
||
"https://shinsei.e-gov.go.jp/top/message/delivery-list",
|
||
wait_until="domcontentloaded",
|
||
)
|
||
try:
|
||
await page.wait_for_load_state("networkidle", timeout=15_000)
|
||
except Exception:
|
||
pass
|
||
|
||
# ─────────────────────────────────────────────────────
|
||
# 診断ステージ2: delivery-list ページの詳細状態を収集
|
||
# ─────────────────────────────────────────────────────
|
||
post = await page.evaluate("""
|
||
() => {
|
||
const b = document.body ? document.body.innerText : '(empty)';
|
||
return {
|
||
url: location.href,
|
||
title: document.title,
|
||
hasLogout: b.includes('ログアウト'),
|
||
hasMyPage: b.includes('マイページ'),
|
||
hasDelivery: b.includes('電子送達'),
|
||
bodyLen: b.length,
|
||
bodyPreview: b.slice(0, 300).replace(/\\n+/g, '|'),
|
||
links: Array.from(document.querySelectorAll('a')).map(a => ({
|
||
t: a.textContent.trim().slice(0, 30),
|
||
h: a.href
|
||
})).slice(0, 20),
|
||
};
|
||
}
|
||
""")
|
||
await _screenshot(page)
|
||
state.message = (
|
||
f"[診断2/goto後] title={post['title']} url={post['url']} "
|
||
f"ログアウト={post['hasLogout']} マイページ={post['hasMyPage']} "
|
||
f"電子送達={post['hasDelivery']} bodyLen={post['bodyLen']} "
|
||
f"body={post['bodyPreview'][:200]}"
|
||
)
|
||
|
||
# 認証状態チェック
|
||
if not post['hasLogout'] and not post['hasMyPage']:
|
||
state.status = EGovStatus.ERROR
|
||
state.message = (
|
||
f"[未認証] delivery-listが未認証状態です。"
|
||
f" goto前: ログアウト={pre.get('hasLogout')} "
|
||
f" goto後: ログアウト={post['hasLogout']} マイページ={post['hasMyPage']}"
|
||
f" body={post['bodyPreview'][:150]}"
|
||
f" links={post['links'][:10]}"
|
||
)
|
||
return
|
||
|
||
current_url = post['url']
|
||
|
||
# ── 通知タイトル（【...】形式）が現れるまで待機 ──
|
||
state.message = "[診断2OK] delivery-listのアイテムを待機中..."
|
||
try:
|
||
await page.wait_for_function(
|
||
"() => Array.from(document.querySelectorAll('a'))"
|
||
".some(a => a.textContent.includes('【') && a.href && !a.href.startsWith('javascript'))",
|
||
timeout=15_000,
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
await _screenshot(page)
|
||
|
||
# ── 最初のアイテムリンクをJSで取得 ─────────────
|
||
detail_href = await page.evaluate("""
|
||
() => {
|
||
const titleLinks = Array.from(document.querySelectorAll('a')).filter(a =>
|
||
a.textContent.trim().startsWith('【') &&
|
||
a.href && !a.href.startsWith('javascript')
|
||
);
|
||
if (titleLinks.length > 0) return titleLinks[0].href;
|
||
|
||
const detailLinks = Array.from(document.querySelectorAll('a[href]')).filter(a =>
|
||
a.href.includes('delivery-detail') || a.href.includes('delivery/detail')
|
||
);
|
||
if (detailLinks.length > 0) return detailLinks[0].href;
|
||
|
||
return null;
|
||
}
|
||
""")
|
||
|
||
if not detail_href:
|
||
all_info = await page.evaluate("""
|
||
() => {
|
||
const b = document.body ? document.body.innerText : '';
|
||
return {
|
||
hrefs: Array.from(document.querySelectorAll('a[href]')).map(a => a.href).slice(0,10),
|
||
bodyPreview: b.slice(0, 300).replace(/\\n+/g, '|'),
|
||
};
|
||
}
|
||
""")
|
||
state.status = EGovStatus.ERROR
|
||
state.message = (
|
||
f"電子送達アイテムなし。URL={current_url} "
|
||
f"body={all_info['bodyPreview'][:200]} "
|
||
f"links={all_info['hrefs']}"
|
||
)
|
||
await _screenshot(page)
|
||
return
|
||
|
||
# ── 詳細ページへ移動 ────────────────────────────
|
||
state.message = "詳細ページへ移動中..."
|
||
await page.goto(detail_href, wait_until="domcontentloaded")
|
||
try:
|
||
await page.wait_for_load_state("networkidle", timeout=10_000)
|
||
except Exception:
|
||
pass
|
||
await _screenshot(page)
|
||
|
||
# ── 通知ファイルをダウンロード ──────────────────
|
||
state.message = "ダウンロードボタンを探しています..."
|
||
try:
|
||
async with page.expect_download(timeout=60_000) as dl_info:
|
||
await page.locator(SEL_DOWNLOAD_BTN).first.click(timeout=15_000)
|
||
download = await dl_info.value
|
||
filename = download.suggested_filename or "egov_download"
|
||
save_path = DOWNLOADS_DIR / filename
|
||
await download.save_as(str(save_path))
|
||
file_size = save_path.stat().st_size
|
||
|
||
await _save_cookies(state._context)
|
||
state.last_download = {
|
||
"filename": filename,
|
||
"path": str(save_path),
|
||
"size": file_size,
|
||
}
|
||
state.status = EGovStatus.LOGGED_IN
|
||
state.message = f"ダウンロード完了: {filename} ({file_size:,} bytes)"
|
||
await _screenshot(page)
|
||
except Exception as e:
|
||
state.status = EGovStatus.ERROR
|
||
state.message = f"ダウンロードに失敗しました: {e}"
|
||
await _screenshot(page)
|
||
|
||
except Exception as exc:
|
||
state.status = EGovStatus.ERROR
|
||
state.message = f"エラー: {exc}"
|
||
|
||
|
||
# ── プライベートヘルパー ─────────────────────────────────────
|
||
|
||
async def _navigate_to_gbizid(page):
|
||
"""ログインページ → GビズID 選択まで自動遷移"""
|
||
# アプローチ1: /top/login へ直接移動
|
||
try:
|
||
await page.goto(
|
||
"https://shinsei.e-gov.go.jp/top/login",
|
||
wait_until="domcontentloaded",
|
||
)
|
||
# Angular レンダリング完了を待機
|
||
try:
|
||
await page.wait_for_load_state("networkidle", timeout=8_000)
|
||
except Exception:
|
||
pass
|
||
await page.wait_for_timeout(2000)
|
||
except Exception:
|
||
pass
|
||
|
||
# /top/login にたどり着けなかった場合: ヘッダーのログインリンクをクリック
|
||
if "login" not in page.url and "keycloak" not in page.url and "gbizid" not in page.url:
|
||
try:
|
||
await page.locator(SEL_LOGIN_LINK).first.click(timeout=8_000)
|
||
await page.wait_for_load_state("domcontentloaded")
|
||
await page.wait_for_timeout(3000)
|
||
except Exception:
|
||
pass
|
||
|
||
await _screenshot(page)
|
||
|
||
# すでに Keycloak/GビズID ページにいる場合はスキップ
|
||
if any(k in page.url for k in ("keycloak", "gbizid", "accounts.g-biz", "gbiz.go.jp")):
|
||
return
|
||
|
||
# GビズID ボタンをクリック（複数セレクタで試行）
|
||
gbizid_selectors = [
|
||
'a:has-text("GビズID")',
|
||
'button:has-text("GビズID")',
|
||
'img[alt*="GビズID"]',
|
||
'[alt*="GビズID"]',
|
||
'a[href*="gbizid"]',
|
||
'a[href*="keycloak"]',
|
||
'a[href*="gbiz"]',
|
||
'[class*="gbiz"]',
|
||
'[id*="gbiz"]',
|
||
]
|
||
clicked = False
|
||
for sel in gbizid_selectors:
|
||
try:
|
||
el = page.locator(sel).first
|
||
await el.wait_for(timeout=3_000, state="visible")
|
||
await el.click()
|
||
await page.wait_for_load_state("domcontentloaded")
|
||
clicked = True
|
||
break
|
||
except Exception:
|
||
continue
|
||
|
||
if not clicked:
|
||
# JS で全 a/button/[role=button] をスキャンして GビズID 関連を探す
|
||
try:
|
||
clicked_js = await page.evaluate("""
|
||
() => {
|
||
const all = Array.from(document.querySelectorAll('a, button, [role="button"]'));
|
||
for (const el of all) {
|
||
const t = (el.textContent || '').trim();
|
||
const h = el.getAttribute('href') || '';
|
||
const alts = Array.from(el.querySelectorAll('img'))
|
||
.map(img => img.alt || '').join(' ');
|
||
if (t.includes('GビズID') || t.includes('gBizID') || t.includes('Gビズ')
|
||
|| h.includes('gbizid') || h.includes('keycloak') || h.includes('gbiz')
|
||
|| alts.includes('GビズID') || alts.includes('ビズID')) {
|
||
el.click();
|
||
return el.tagName + ':' + t.slice(0, 30) + ':' + h.slice(0, 60);
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
""")
|
||
if clicked_js:
|
||
await page.wait_for_load_state("domcontentloaded")
|
||
await page.wait_for_timeout(2000)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
async def _is_logged_in(page) -> bool:
|
||
try:
|
||
content = await page.content()
|
||
return any(m in content for m in LOGGEDIN_MARKERS)
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
async def _load_context(browser):
|
||
"""保存済み Cookie からセッションを復元"""
|
||
if COOKIES_PATH.exists():
|
||
try:
|
||
with open(COOKIES_PATH) as f:
|
||
storage = json.load(f)
|
||
return await browser.new_context(storage_state=storage)
|
||
except Exception:
|
||
pass
|
||
return await browser.new_context()
|
||
|
||
|
||
async def _save_cookies(context):
|
||
"""セッション Cookie をファイルに保存"""
|
||
EGOV_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||
storage = await context.storage_state()
|
||
with open(COOKIES_PATH, "w") as f:
|
||
json.dump(storage, f)
|
||
|
||
|
||
async def _screenshot(page):
|
||
"""ページのスクリーンショットを base64 で state に保存"""
|
||
try:
|
||
data = await page.screenshot(type="jpeg", quality=55)
|
||
state.screenshot_b64 = base64.b64encode(data).decode()
|
||
except Exception:
|
||
pass
|