#!/usr/bin/env python3 """ serp_checker.py ================ Google SERP Rank Tracker for TOP EDU PREP (topeduprep.uk) Queries the Google Programmable Search Engine (Custom Search JSON API) for a list of target keywords and records the ranking position of a target domain within the top N results (default: top 30, i.e. pages 1-3). Setup ----- 1. Create a Programmable Search Engine at https://programmablesearchengine.google.com/ - Set it to "Search the entire web". - Copy the Search Engine ID (cx). 2. Enable the "Custom Search API" in Google Cloud Console and create an API key: https://console.cloud.google.com/apis/library/customsearch.googleapis.com 3. Export credentials as environment variables (recommended, keeps keys out of code): export GOOGLE_CSE_API_KEY="your_api_key_here" export GOOGLE_CSE_ID="your_search_engine_id_here" 4. Install dependencies: pip install requests pandas Usage ----- python serp_checker.py python serp_checker.py --domain topeduprep.uk --max-results 30 --out rankings.csv python serp_checker.py --keywords "국제학교 SAT 준비 몇학년부터" "IB 디플로마 SAT 병행" Notes ----- - The free tier of the Custom Search JSON API allows 100 queries/day. Each "page" of 10 results = 1 query, so checking 30 results per keyword costs 3 queries per keyword. - Results are cached to CSV with a timestamp so historical rank trends can be tracked over time by re-running this script (e.g. via cron / Task Scheduler) and appending to a master log. """ from __future__ import annotations import argparse import os import sys import time from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Optional from urllib.parse import urlparse import pandas as pd import requests GOOGLE_CSE_ENDPOINT = "https://www.googleapis.com/customsearch/v1" RESULTS_PER_PAGE = 10 # fixed by the Google CSE API DEFAULT_MAX_RESULTS = 30 # top 3 pages REQUEST_TIMEOUT_SECONDS = 15 RATE_LIMIT_SLEEP_SECONDS = 1.0 # polite delay between API calls # Default keyword set for TOP EDU PREP. Override with --keywords on the CLI. DEFAULT_KEYWORDS = [ "국제학교 SAT 준비 몇학년부터", "IB 디플로마 SAT 병행", "국제학교 AP 준비", "SAT 수학 학원", "IB AA HL 과외", "압구정 SAT 학원", "잠실 IB 학원", "센텀시티 AP 학원", ] DEFAULT_DOMAIN = "topeduprep.uk" @dataclass class RankResult: keyword: str domain: str rank: Optional[int] # 1-indexed position, None if not found matched_url: Optional[str] = None matched_title: Optional[str] = None results_checked: int = 0 checked_at: str = field( default_factory=lambda: datetime.now(timezone.utc).isoformat(timespec="seconds") ) error: Optional[str] = None def normalize_domain(url_or_domain: str) -> str: """Strip scheme/www/path so 'https://www.topeduprep.uk/blog' -> 'topeduprep.uk'.""" candidate = url_or_domain.strip().lower() if "://" not in candidate: candidate = "https://" + candidate netloc = urlparse(candidate).netloc or candidate if netloc.startswith("www."): netloc = netloc[4:] return netloc def fetch_search_page( api_key: str, cse_id: str, keyword: str, start_index: int ) -> dict: """Fetch a single page (10 results) of Google search results.""" params = { "key": api_key, "cx": cse_id, "q": keyword, "start": start_index, # 1-indexed: 1, 11, 21, ... "num": RESULTS_PER_PAGE, "gl": "kr", # bias results toward Korea "hl": "ko", # interface/response language } response = requests.get( GOOGLE_CSE_ENDPOINT, params=params, timeout=REQUEST_TIMEOUT_SECONDS ) response.raise_for_status() return response.json() def find_rank_for_keyword( api_key: str, cse_id: str, keyword: str, target_domain: str, max_results: int = DEFAULT_MAX_RESULTS, ) -> RankResult: """ Search Google for `keyword` and return the 1-indexed rank of the first result whose URL belongs to `target_domain`, scanning up to max_results. """ target_domain = normalize_domain(target_domain) checked = 0 pages_needed = (max_results + RESULTS_PER_PAGE - 1) // RESULTS_PER_PAGE for page in range(pages_needed): start_index = page * RESULTS_PER_PAGE + 1 try: data = fetch_search_page(api_key, cse_id, keyword, start_index) except requests.exceptions.HTTPError as http_err: return RankResult( keyword=keyword, domain=target_domain, rank=None, results_checked=checked, error=f"HTTP error: {http_err} — response: {getattr(http_err.response, 'text', '')[:300]}", ) except requests.exceptions.RequestException as req_err: return RankResult( keyword=keyword, domain=target_domain, rank=None, results_checked=checked, error=f"Request error: {req_err}", ) items = data.get("items", []) if not items: # No more results available (or zero results for this keyword) break for offset, item in enumerate(items): checked += 1 item_url = item.get("link", "") item_domain = normalize_domain(item_url) if item_domain == target_domain or item_domain.endswith("." + target_domain): absolute_rank = start_index + offset return RankResult( keyword=keyword, domain=target_domain, rank=absolute_rank, matched_url=item_url, matched_title=item.get("title"), results_checked=checked, ) if checked >= max_results: break if checked >= max_results: break time.sleep(RATE_LIMIT_SLEEP_SECONDS) # Domain not found within max_results return RankResult( keyword=keyword, domain=target_domain, rank=None, results_checked=checked, ) def run_rank_check( keywords: list[str], domain: str, api_key: str, cse_id: str, max_results: int = DEFAULT_MAX_RESULTS, ) -> pd.DataFrame: """Run rank checks for every keyword and return a tidy DataFrame.""" rows: list[dict] = [] for idx, keyword in enumerate(keywords, start=1): print(f"[{idx}/{len(keywords)}] Checking: {keyword!r} ...", file=sys.stderr) result = find_rank_for_keyword( api_key=api_key, cse_id=cse_id, keyword=keyword, target_domain=domain, max_results=max_results, ) rows.append( { "keyword": result.keyword, "domain": result.domain, "rank": result.rank if result.rank is not None else f"Not in top {max_results}", "matched_url": result.matched_url or "", "matched_title": result.matched_title or "", "results_checked": result.results_checked, "checked_at_utc": result.checked_at, "error": result.error or "", } ) if result.error: print(f" -> ERROR: {result.error}", file=sys.stderr) elif result.rank: print(f" -> Found at rank #{result.rank}", file=sys.stderr) else: print(f" -> Not found in top {max_results} results", file=sys.stderr) time.sleep(RATE_LIMIT_SLEEP_SECONDS) df = pd.DataFrame(rows) # Sort so found rankings (numeric) come first, best rank at top df["_sort_rank"] = pd.to_numeric( df["rank"], errors="coerce" ).fillna(float("inf")) df = df.sort_values("_sort_rank").drop(columns="_sort_rank").reset_index(drop=True) return df def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Track Google SERP rankings for TOP EDU PREP target keywords." ) parser.add_argument( "--domain", default=DEFAULT_DOMAIN, help=f"Target domain to search for in results (default: {DEFAULT_DOMAIN})", ) parser.add_argument( "--keywords", nargs="+", default=None, help="One or more keywords to check (space-separated, quote multi-word phrases). " "Defaults to the built-in TOP EDU PREP keyword set.", ) parser.add_argument( "--max-results", type=int, default=DEFAULT_MAX_RESULTS, help=f"How many organic results to scan per keyword (default: {DEFAULT_MAX_RESULTS})", ) parser.add_argument( "--out", default=None, help="Output CSV path. Defaults to serp_rankings_.csv in the current directory.", ) parser.add_argument( "--api-key", default=None, help="Google Custom Search API key. Defaults to the GOOGLE_CSE_API_KEY env var.", ) parser.add_argument( "--cse-id", default=None, help="Google Custom Search Engine ID (cx). Defaults to the GOOGLE_CSE_ID env var.", ) return parser.parse_args() def main() -> None: args = parse_args() api_key = args.api_key or os.environ.get("GOOGLE_CSE_API_KEY") cse_id = args.cse_id or os.environ.get("GOOGLE_CSE_ID") if not api_key or not cse_id: print( "ERROR: Missing credentials.\n" "Set GOOGLE_CSE_API_KEY and GOOGLE_CSE_ID environment variables, " "or pass --api-key and --cse-id.\n\n" "See the module docstring at the top of this file for setup instructions.", file=sys.stderr, ) sys.exit(1) keywords = args.keywords or DEFAULT_KEYWORDS df = run_rank_check( keywords=keywords, domain=args.domain, api_key=api_key, cse_id=cse_id, max_results=args.max_results, ) print("\n=== SERP Ranking Summary ===") print(df.to_string(index=False)) out_path = args.out if not out_path: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") out_path = f"serp_rankings_{timestamp}.csv" df.to_csv(out_path, index=False, encoding="utf-8-sig") print(f"\nSaved results to: {out_path}", file=sys.stderr) if __name__ == "__main__": main()