#!/usr/bin/env python3 """ Crawl the legacy shodh.net static content pages (Joomla com_content articles) and generate the CMS seed migration. Outputs: - legacy-import/content.json raw extracted page bodies - src/main/resources/db/migration/V16__seed_legacy_pages.sql one CMS row per page: section / group 'page' / field 'body' (bodies are bilingual on the legacy site, so value_hi = value_en) V15__cms_page_sections.sql (hand-written) must add the enum values first — Postgres cannot use a new enum value in the migration that creates it. Run from shodh-sanchayan-api/: python3 scripts/legacy_content_crawl.py """ import html as htmllib import json import re import time import unicodedata import urllib.request from pathlib import Path BASE = "https://shodh.net" DELAY_SECONDS = 0.6 USER_AGENT = "ShodhSanchayanMigration/1.0 (content import; contact info@shodh.net)" REPO_ROOT = Path(__file__).resolve().parent.parent OUT_DIR = REPO_ROOT / "legacy-import" MIGRATION = REPO_ROOT / "src/main/resources/db/migration/V16__seed_legacy_pages.sql" # (article id, Itemid, ContentSection, title_hi, title_en) — inventory §1. # Pages with a live route today; the trailing entries are captured for later use. PAGES = [ (22, 43, "JOURNAL_AIMS", "लक्ष्य एवं उद्देश्य", "Aims & Objectives"), (5, 44, "JOURNAL_ABOUT", "पत्रिका के बारे में", "About the Journal"), (56, 151, "JOURNAL_IMPACT_FACTOR", "प्रभाव कारक", "Impact Factor"), (6, 45, "JOURNAL_EDITORIAL_BOARD", "संपादकीय बोर्ड", "Editorial Board"), (39, 48, "JOURNAL_SPECIAL_ISSUE", "विशेष अंक", "Special Issue"), (7, 42, "AUTHOR_GUIDELINES", "प्रस्तुति दिशानिर्देश", "Submission Guidelines"), (8, 80, "AUTHOR_SUBSCRIPTION", "सदस्यता शुल्क", "Subscription Fee"), (25, 50, "AUTHOR_PUBLICATION_FEE", "प्रकाशन शुल्क", "Publication Fee"), (26, 49, "AUTHOR_DEADLINES", "समय-सीमाएँ", "Submission Deadlines"), (24, 51, "AUTHOR_PEER_REVIEW", "समीक्षा प्रक्रिया", "Peer Review Process"), (23, 53, "AUTHOR_CALL_FOR_PAPERS", "शोधपत्र आमंत्रण", "Call for Papers"), (30, 58, "INFO_DOCTORAL_COLLOQUIUM", "डॉक्टरल संगोष्ठी", "Doctoral Colloquium"), (31, 59, "INFO_WORKSHOPS", "शैक्षणिक कार्यशाला", "Academic Workshops"), (32, 60, "INFO_TRAINING", "शैक्षणिक प्रशिक्षण", "Academic Training"), (9, 22, "INFO_GRANT_INSTITUTIONS", "शोध अनुदानदाता संस्थाएँ", "Grant Institutions"), (20, 82, "DISCLAIMER", "अस्वीकरण", "Disclaimer"), # Captured to content.json only (no dedicated page yet): (33, 64, None, "अपना शोधपत्र प्रकाशित करें", "Publish your Research Paper"), (34, 65, None, "अपना शोध सारांश प्रकाशित करें", "Publish your Research Abstract"), (36, 67, None, "अपनी परियोजना रिपोर्ट प्रकाशित करें", "Publish your Project Report"), (21, 10, None, "हमारे बारे में", "About Us"), (18, 41, None, "संपर्क", "Contact Us"), ] ALLOWED_TAGS = {"p", "br", "ul", "ol", "li", "table", "tbody", "thead", "tr", "td", "th", "h1", "h2", "h3", "h4", "strong", "b", "em", "i", "u", "a", "blockquote", "span", "div", "hr"} def fetch(url: str) -> str: time.sleep(DELAY_SECONDS) req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(req, timeout=60) as resp: return resp.read().decode("utf-8", errors="replace") def extract_body(page_html: str) -> str | None: """Article body = the second contentpaneopen table on the page.""" m = re.search( r'.*?
\s*' r'(.*?)
', page_html, re.DOTALL) return m.group(1) if m else None def sanitize(body: str) -> str: # unwrap the outer that contentpaneopen tables put around the body body = re.sub(r"^\s*]*>\s*]*>", "", body, flags=re.IGNORECASE) body = re.sub(r"\s*\s*$", "", body, flags=re.IGNORECASE) body = re.sub(r"", "", body, flags=re.DOTALL) body = re.sub(r"", "", body, flags=re.DOTALL | re.IGNORECASE) body = re.sub(r"", "", body, flags=re.DOTALL | re.IGNORECASE) # strip Joomla chrome cells (print/pdf/email buttons, author/date rows) body = re.sub(r']*class="(?:buttonheading|createdate|modifydate)"[^>]*>.*?', "", body, flags=re.DOTALL) def tag_filter(m): closing, name = m.group(1), m.group(2).lower() if name not in ALLOWED_TAGS: return "" if closing: return f"" if name == "a": href = re.search(r'href="([^"]*)"', m.group(0)) url = htmllib.unescape(href.group(1)) if href else "" if url.startswith("/") or url.startswith("index.php"): url = BASE + ("/" if not url.startswith("/") else "") + url elif url.startswith(("media/", "ugc_pdf/", "phocadownload/")): url = f"{BASE}/{url}" return f'' if name == "br" or name == "hr": return f"<{name}/>" return f"<{name}>" body = re.sub(r"<(/?)([a-zA-Z0-9]+)(?:\s[^>]*)?>", tag_filter, body) # collapse whitespace and empty paragraphs body = re.sub(r"

\s*(?: |\s)*\s*

", "", body) body = re.sub(r"[ \t]+", " ", body) body = re.sub(r"\n{3,}", "\n\n", body) return unicodedata.normalize("NFC", body.strip()) def sql_quote(text: str) -> str: return "'" + text.replace("'", "''") + "'" def main(): OUT_DIR.mkdir(exist_ok=True) captured, problems = [], [] for article_id, itemid, section, title_hi, title_en in PAGES: url = f"{BASE}/index.php?option=com_content&view=article&id={article_id}&Itemid={itemid}" print(f"[page] id={article_id} {title_en} …", flush=True) try: page = fetch(url) except Exception as e: problems.append(f"id={article_id} {title_en}: fetch FAILED: {e}") continue raw = extract_body(page) if not raw: problems.append(f"id={article_id} {title_en}: body not found in page") continue body = sanitize(raw) text_len = len(re.sub(r"<[^>]+>", "", body).strip()) if text_len < 40: problems.append(f"id={article_id} {title_en}: body suspiciously short ({text_len} chars)") captured.append({ "articleId": article_id, "itemid": itemid, "section": section, "titleHi": title_hi, "titleEn": title_en, "textChars": text_len, "bodyHtml": body, }) (OUT_DIR / "content.json").write_text( json.dumps({"crawledAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "problems": problems, "pages": captured}, ensure_ascii=False, indent=1), encoding="utf-8") # ---- generate the seed migration for pages that have a section ---- lines = [ "-- Generated by scripts/legacy_content_crawl.py — one-time seed of the", "-- legacy shodh.net static pages into the CMS. After this lands, the", "-- admin editor at /admin/page-content is the source of truth.", "-- Legacy bodies are bilingual (Hindi and English interleaved), so the", "-- same body seeds both value columns.", "", ] for page in captured: if not page["section"]: continue # A near-empty legacy body (e.g. Special Issue is just prev/next links) # seeds as empty so the UI shows its graceful fallback instead of junk. body_html = page["bodyHtml"] if page["textChars"] >= 40 else "" body = sql_quote(body_html) lines.append( "INSERT INTO cms_content (section, group_key, field_key, title_hi, title_en, " "value_hi, value_en, field_type, sort_order)\n" f"VALUES ('{page['section']}', 'page', 'body', " f"{sql_quote(page['titleHi'])}, {sql_quote(page['titleEn'])}, " f"{body}, {body}, 'html', 1)\n" "ON CONFLICT (section, group_key, field_key) DO NOTHING;\n") MIGRATION.write_text("\n".join(lines), encoding="utf-8") seeded = sum(1 for p in captured if p["section"]) print(f"\nDone. {len(captured)} pages captured, {seeded} seeded to {MIGRATION.name}, " f"{len(problems)} problem(s).") for p in problems: print(" !", p) if __name__ == "__main__": main()