""" serp_checker.py ================ Keyword SERP Rank Checker for TOP EDU PREP (topeduprep.uk) Queries the Google Programmable Search Engine (Custom Search JSON API) for a list of target keywords and reports the ranking position of a target domain within the results, exporting a clean pandas DataFrame / CSV. SETUP ----- 1. Create a Custom Search Engine at https://programmablesearchengine.google.com/ - Set it to search the entire web (not just a specific site). - Copy the "Search engine ID" (this is your GOOGLE_CSE_ID). 2. Enable the "Custom Search API" in Google Cloud Console and create an API key (this is your GOOGLE_API_KEY). 3. Export both as environment variables (recommended, keeps secrets out of code): export GOOGLE_API_KEY="your-api-key-here" export GOOGLE_CSE_ID="your-search-engine-id-here" 4. Install dependencies: pip install requests pandas --break-system-packages USAGE ----- python serp_checker.py python serp_checker.py --domain topeduprep.uk --pages 3 --out results.csv python serp_checker.py --keywords-file keywords.txt NOTES ----- - The free tier of the Custom Search JSON API allows 100 queries/day. Each "page" of 10 results consumes 1 query, so checking 3 pages (30 results) per keyword uses 3 queries per keyword. - Google's public SERPs and the Custom Search API do not return identical rankings (the API is scoped to your CSE's index settings), so treat results as a directional proxy for real SERP position, not a pixel-perfect mirror. """ import argparse import os import sys import time from datetime import datetime, timezone from urllib.parse import urlparse import pandas as pd import requests API_ENDPOINT = "https://www.googleapis.com/customsearch/v1" RESULTS_PER_PAGE = 10 # Google Custom Search API max per request REQUEST_DELAY_SECONDS = 1.0 # polite delay between API calls to avoid rate-limit errors DEFAULT_KEYWORDS = [ "국제학교 SAT 준비 몇학년부터", "IB 디플로마 SAT 병행", "Digital SAT prep international school", "AP Calculus tutor Seoul", "IB Math AA tutor online", "SAT academy Apgujeong", ] def normalize_domain(url: str) -> str: """Strip protocol, www prefix, and trailing slash from a URL/domain string.""" parsed = urlparse(url if "://" in url else f"//{url}", scheme="") host = (parsed.netloc or parsed.path).lower() if host.startswith("www."): host = host[4:] return host.rstrip("/") def fetch_serp_page(query: str, api_key: str, cse_id: str, start_index: int) -> dict: """ Fetch a single page (10 results) of Google Custom Search results. `start_index` is 1-based per Google's API (1, 11, 21, ...). """ params = { "key": api_key, "cx": cse_id, "q": query, "start": start_index, "num": RESULTS_PER_PAGE, } response = requests.get(API_ENDPOINT, params=params, timeout=15) if response.status_code != 200: raise RuntimeError( f"Google Custom Search API error {response.status_code} for query " f"'{query}' (start={start_index}): {response.text[:300]}" ) return response.json() def find_rank_for_keyword(keyword: str, target_domain: str, api_key: str, cse_id: str, max_pages: int) -> dict: """ Search up to `max_pages` pages (10 results each) for `keyword` and return the first position at which `target_domain` appears, along with the matching URL and title. Returns rank=None if not found within scanned pages. """ target_domain = normalize_domain(target_domain) overall_position = 0 for page in range(max_pages): start_index = page * RESULTS_PER_PAGE + 1 try: data = fetch_serp_page(keyword, api_key, cse_id, start_index) except RuntimeError as err: return { "keyword": keyword, "rank": None, "matched_url": None, "matched_title": None, "pages_scanned": page, "error": str(err), } items = data.get("items", []) if not items: # No more results available (end of index or zero results for this keyword) break for item in items: overall_position += 1 result_domain = normalize_domain(item.get("link", "")) if result_domain == target_domain or result_domain.endswith(f".{target_domain}"): return { "keyword": keyword, "rank": overall_position, "matched_url": item.get("link"), "matched_title": item.get("title"), "pages_scanned": page + 1, "error": None, } time.sleep(REQUEST_DELAY_SECONDS) # be polite between page requests return { "keyword": keyword, "rank": None, "matched_url": None, "matched_title": None, "pages_scanned": max_pages, "error": None, } def load_keywords_from_file(path: str) -> list: with open(path, "r", encoding="utf-8") as f: return [line.strip() for line in f if line.strip()] def run_rank_check(keywords: list, domain: str, api_key: str, cse_id: str, max_pages: int) -> pd.DataFrame: rows = [] total = len(keywords) for i, keyword in enumerate(keywords, start=1): print(f"[{i}/{total}] Checking rank for: {keyword!r} ...", flush=True) result = find_rank_for_keyword(keyword, domain, api_key, cse_id, max_pages) result["domain"] = domain result["checked_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds") if result["error"]: print(f" -> error: {result['error']}") elif result["rank"]: print(f" -> found at position #{result['rank']} ({result['matched_url']})") else: print(f" -> not found in top {result['pages_scanned'] * RESULTS_PER_PAGE} results") rows.append(result) time.sleep(REQUEST_DELAY_SECONDS) # polite delay between distinct keyword searches df = pd.DataFrame(rows, columns=[ "keyword", "domain", "rank", "matched_url", "matched_title", "pages_scanned", "checked_at", "error", ]) df = df.sort_values(by="rank", na_position="last").reset_index(drop=True) return df def parse_args(): parser = argparse.ArgumentParser( description="Track Google SERP ranking position of a domain for a list of target keywords." ) parser.add_argument( "--domain", default="topeduprep.uk", help="Target domain to search for in results (default: topeduprep.uk)", ) parser.add_argument( "--pages", type=int, default=3, help="Number of result pages (10 results each) to scan per keyword (default: 3, i.e. top 30)", ) parser.add_argument( "--keywords-file", default=None, help="Path to a plain-text file with one keyword per line. If omitted, uses a built-in default list.", ) parser.add_argument( "--out", default=None, help="Output CSV path. If omitted, auto-names as serp_results_.csv", ) parser.add_argument( "--api-key", default=None, help="Google API key. Falls back to the GOOGLE_API_KEY environment variable.", ) parser.add_argument( "--cse-id", default=None, help="Google Custom Search Engine ID. Falls back to the GOOGLE_CSE_ID environment variable.", ) return parser.parse_args() def main(): args = parse_args() api_key = args.api_key or os.environ.get("GOOGLE_API_KEY") cse_id = args.cse_id or os.environ.get("GOOGLE_CSE_ID") if not api_key or not cse_id: print( "Missing credentials. Set GOOGLE_API_KEY and GOOGLE_CSE_ID environment variables,\n" "or pass --api-key and --cse-id explicitly. See the module docstring for setup steps.", file=sys.stderr, ) sys.exit(1) keywords = load_keywords_from_file(args.keywords_file) if args.keywords_file else DEFAULT_KEYWORDS df = run_rank_check( keywords=keywords, domain=args.domain, api_key=api_key, cse_id=cse_id, max_pages=args.pages, ) out_path = args.out or f"serp_results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv" df.to_csv(out_path, index=False, encoding="utf-8-sig") # utf-8-sig keeps Korean text readable in Excel print("\n=== Rank Check Summary ===") print(df.to_string(index=False)) print(f"\nSaved {len(df)} rows to {out_path}") if __name__ == "__main__": main()