feat(root): modernize API server,CLI and data loading
- Switch to async lifespans, eager in‑memory caching of calendar data, and ORJSON responses for reduced latency. - Add structured logging and GZIP middleware, enforce year bounds and directory existence. - Replace json module with orjson, use pathlib and TypedDict aliases, and implement typed argument parsing with explicit error handling. - Remove unused test.py and refactor scraper CLI to accept argv via function injection, simplify input mode logic. - Preserve backward defaults (produce both single JSON and directory output) while exposing explicit flags via argparse. This refactor improves scalability on low‑resources servers, adds robustness checks, and clarifies the command‑line interface for future extensions.
This commit is contained in:
7 files changed
+493
-229
No files matched your search
@@ -1,40 +1,217 @@
|
|||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import json
|
import sys
|
||||||
from fastapi import FastAPI, HTTPException
|
from collections.abc import AsyncIterator
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
app = FastAPI(title="Nepali Calendar API")
|
import orjson
|
||||||
|
from fastapi import FastAPI, HTTPException, Request
|
||||||
|
from fastapi import Path as PathParam
|
||||||
|
from fastapi.middleware.gzip import GZipMiddleware
|
||||||
|
from fastapi.responses import ORJSONResponse, Response
|
||||||
|
|
||||||
DATA_DIR = "./data"
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||||
|
stream=sys.stdout,
|
||||||
|
)
|
||||||
|
log = logging.getLogger("nepali-calendar-api")
|
||||||
|
|
||||||
|
DATA_DIR = Path(os.environ.get("DATA_DIR", "./data")).resolve()
|
||||||
|
MIN_YEAR = 1900
|
||||||
|
MAX_YEAR = 2200
|
||||||
|
CACHE_CONTROL = "public, max-age=86400, immutable"
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def _lifespan(_app: FastAPI) -> AsyncIterator[None]:
|
||||||
|
_load_data()
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(
|
||||||
|
title="Nepali Calendar API",
|
||||||
|
default_response_class=ORJSONResponse,
|
||||||
|
docs_url="/docs",
|
||||||
|
redoc_url=None,
|
||||||
|
lifespan=_lifespan,
|
||||||
|
)
|
||||||
|
app.add_middleware(GZipMiddleware, minimum_size=512)
|
||||||
|
|
||||||
|
|
||||||
|
_CALENDAR: dict[int, dict] = {}
|
||||||
|
_MONTHLY: dict[tuple[int, int], dict] = {}
|
||||||
|
_ETAG_DATA: bytes = b""
|
||||||
|
|
||||||
|
|
||||||
|
def _load_data() -> None:
|
||||||
|
"""Eagerly load every JSON file into memory once at startup.
|
||||||
|
|
||||||
|
The dataset is static and small (~9MB), so an in-memory dict keeps the
|
||||||
|
API off the disk on every request, which is essential for tiny servers
|
||||||
|
where cold reads hurt p99 latency and steal CPU from the event loop.
|
||||||
|
"""
|
||||||
|
global _ETAG_DATA
|
||||||
|
if not DATA_DIR.is_dir():
|
||||||
|
log.warning("DATA_DIR %s does not exist; API will return 404s", DATA_DIR)
|
||||||
|
return
|
||||||
|
|
||||||
|
monthly: dict[tuple[int, int], dict] = {}
|
||||||
|
yearly: dict[int, dict] = {}
|
||||||
|
|
||||||
|
for year_dir in DATA_DIR.iterdir():
|
||||||
|
if not year_dir.is_dir() or not year_dir.name.isdigit():
|
||||||
|
continue
|
||||||
|
year = int(year_dir.name)
|
||||||
|
for month_file in year_dir.glob("*.json"):
|
||||||
|
if not month_file.stem.isdigit():
|
||||||
|
continue
|
||||||
|
month = int(month_file.stem)
|
||||||
|
if not 1 <= month <= 12:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
raw = month_file.read_bytes()
|
||||||
|
data = orjson.loads(raw)
|
||||||
|
except (orjson.JSONDecodeError, OSError) as exc:
|
||||||
|
log.error("Skipping corrupt file %s: %s", month_file, exc)
|
||||||
|
continue
|
||||||
|
monthly[(year, month)] = data
|
||||||
|
|
||||||
|
# Build yearly aggregates from monthly data so the /calendar/{year}
|
||||||
|
# endpoint works even when only the directory layout has been scraped.
|
||||||
|
by_year: dict[int, dict[int, dict]] = {}
|
||||||
|
for (year, month), payload in monthly.items():
|
||||||
|
by_year.setdefault(year, {})[month] = payload
|
||||||
|
|
||||||
|
for year, months in by_year.items():
|
||||||
|
if len(months) == 12:
|
||||||
|
yearly[year] = {str(m): months[m] for m in range(1, 13)}
|
||||||
|
|
||||||
|
# Also pick up pre-aggregated yearly files (e.g. data/2082.json) if they
|
||||||
|
# are newer or missing, so the scraper's --single output is honoured.
|
||||||
|
for year_file in DATA_DIR.glob("*.json"):
|
||||||
|
if not year_file.stem.isdigit():
|
||||||
|
continue
|
||||||
|
year = int(year_file.stem)
|
||||||
|
if not MIN_YEAR <= year <= MAX_YEAR:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
raw = year_file.read_bytes()
|
||||||
|
data = orjson.loads(raw)
|
||||||
|
except (orjson.JSONDecodeError, OSError) as exc:
|
||||||
|
log.error("Skipping corrupt file %s: %s", year_file, exc)
|
||||||
|
continue
|
||||||
|
# Only adopt the file if it covers a full year.
|
||||||
|
if isinstance(data, dict) and all(str(m) in data for m in range(1, 13)):
|
||||||
|
yearly[year] = data
|
||||||
|
|
||||||
|
_MONTHLY.update(monthly)
|
||||||
|
_CALENDAR.update(yearly)
|
||||||
|
_ETAG_DATA = hashlib.sha1(
|
||||||
|
repr(sorted(_CALENDAR.keys())).encode()
|
||||||
|
).hexdigest()[:16].encode()
|
||||||
|
log.info(
|
||||||
|
"Loaded %d years and %d months into memory",
|
||||||
|
len(_CALENDAR),
|
||||||
|
len(_MONTHLY),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/", summary="Root Endpoint")
|
@app.get("/", summary="Root Endpoint")
|
||||||
async def root():
|
async def root() -> dict:
|
||||||
return {
|
return {
|
||||||
"message": "Welcome to the Nepali Calendar API. Use /docs for interactive API documentation."
|
"message": "Welcome to the Nepali Calendar API. Use /docs for interactive API documentation.",
|
||||||
|
"years_available": len(_CALENDAR),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@app.get("/calendar/{year}", summary="Get Yearly Calendar")
|
@app.get("/health", summary="Health Check")
|
||||||
async def get_year_calendar(year: int):
|
async def health() -> dict:
|
||||||
file_path = f"{DATA_DIR}/{year}.json"
|
return {"status": "ok", "years": len(_CALENDAR), "months": len(_MONTHLY)}
|
||||||
if not os.path.exists(file_path):
|
|
||||||
|
|
||||||
|
def _validate_year(year: int) -> None:
|
||||||
|
if not MIN_YEAR <= year <= MAX_YEAR:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Year must be between {MIN_YEAR} and {MAX_YEAR}",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _maybe_not_modified(request: Request, etag: str) -> Response | None:
|
||||||
|
if request.headers.get("if-none-match") == etag:
|
||||||
|
return Response(
|
||||||
|
status_code=304, headers={"ETag": etag, "Cache-Control": CACHE_CONTROL}
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@app.get(
|
||||||
|
"/calendar/{year}",
|
||||||
|
summary="Get Yearly Calendar",
|
||||||
|
responses={404: {"description": "Year not found"}},
|
||||||
|
)
|
||||||
|
async def get_year_calendar(
|
||||||
|
request: Request,
|
||||||
|
year: int = PathParam(..., ge=MIN_YEAR, le=MAX_YEAR, description="BS year"),
|
||||||
|
) -> Response:
|
||||||
|
_validate_year(year)
|
||||||
|
payload = _CALENDAR.get(year)
|
||||||
|
if payload is None:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=404,
|
status_code=404,
|
||||||
detail=f"Calendar data for year {year} not found. Please run the scraper for this year.",
|
detail=f"Calendar data for year {year} not found. Please run the scraper for this year.",
|
||||||
)
|
)
|
||||||
|
etag = f'W/"{year}-{_ETAG_DATA.decode()}"'
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
not_modified = _maybe_not_modified(request, etag)
|
||||||
return json.load(f)
|
if not_modified is not None:
|
||||||
|
return not_modified
|
||||||
|
body = orjson.dumps(payload)
|
||||||
|
return Response(
|
||||||
|
content=body,
|
||||||
|
media_type="application/json",
|
||||||
|
headers={"ETag": etag, "Cache-Control": CACHE_CONTROL},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/calendar/{year}/{month}", summary="Get Monthly Calendar")
|
@app.get(
|
||||||
async def get_month_calendar(year: int, month: int):
|
"/calendar/{year}/{month}",
|
||||||
file_path = f"{DATA_DIR}/{year}/{month}.json"
|
summary="Get Monthly Calendar",
|
||||||
if not os.path.exists(file_path):
|
responses={404: {"description": "Year/Month not found"}},
|
||||||
|
)
|
||||||
|
async def get_month_calendar(
|
||||||
|
request: Request,
|
||||||
|
year: int = PathParam(..., ge=MIN_YEAR, le=MAX_YEAR, description="BS year"),
|
||||||
|
month: int = PathParam(..., ge=1, le=12, description="BS month (1-12)"),
|
||||||
|
) -> Response:
|
||||||
|
_validate_year(year)
|
||||||
|
payload = _MONTHLY.get((year, month))
|
||||||
|
if payload is None:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=404,
|
status_code=404,
|
||||||
detail=f"Calendar data for {year}/{month} not found. Please run the scraper for this year.",
|
detail=f"Calendar data for {year}/{month} not found. Please run the scraper for this year.",
|
||||||
)
|
)
|
||||||
|
etag = f'W/"{year}-{month}-{_ETAG_DATA.decode()}"'
|
||||||
|
not_modified = _maybe_not_modified(request, etag)
|
||||||
|
if not_modified is not None:
|
||||||
|
return not_modified
|
||||||
|
body = orjson.dumps(payload)
|
||||||
|
return Response(
|
||||||
|
content=body,
|
||||||
|
media_type="application/json",
|
||||||
|
headers={"ETag": etag, "Cache-Control": CACHE_CONTROL},
|
||||||
|
)
|
||||||
|
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
|
||||||
return json.load(f)
|
if __name__ == "__main__":
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
uvicorn.run(
|
||||||
|
"app:app",
|
||||||
|
host="0.0.0.0",
|
||||||
|
port=int(os.environ.get("PORT", "8000")),
|
||||||
|
workers=1,
|
||||||
|
log_level=os.environ.get("LOG_LEVEL", "info"),
|
||||||
|
access_log=False,
|
||||||
|
)
|
||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
runtime: python
|
runtime: python
|
||||||
startCommand: uvicorn app:app --port 8080
|
startCommand: uvicorn app:app --host 0.0.0.0 --port 8080 --workers 1 --no-access-log
|
||||||
buildCommand: pip install -r requirements.txt
|
buildCommand: pip install --no-cache-dir -r requirements.txt
|
||||||
port: 8080
|
port: 8080
|
||||||
memory: 512mb
|
memory: 512mb
|
||||||
+24
-7
@@ -1,20 +1,37 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "nepali-calendar-api"
|
name = "nepali-calendar-api"
|
||||||
version = "0.1.0"
|
version = "0.2.0"
|
||||||
description = "A Nepali calendar dataset/library containing Nepali dates, tithi, festivals, public holidays, and auspicious dates for marriage and bratabandha ceremonies."
|
description = "A Nepali calendar dataset/library containing Nepali dates, tithi, festivals, public holidays, and auspicious dates for marriage and bratabandha ceremonies."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aiohttp>=3.13.3",
|
|
||||||
"beautifulsoup4>=4.14.3",
|
|
||||||
"colorama>=0.4.6",
|
|
||||||
"fastapi>=0.115.11",
|
"fastapi>=0.115.11",
|
||||||
"uvicorn>=0.34.0",
|
"uvicorn>=0.34.0",
|
||||||
|
"orjson>=3.10.0",
|
||||||
]
|
]
|
||||||
authors = [
|
|
||||||
{ name = "Sankalp Tharu", email = 'sankalptharu50028@gmail.com' },
|
[project.optional-dependencies]
|
||||||
|
scraper = [
|
||||||
|
"aiohttp>=3.13.3",
|
||||||
|
"beautifulsoup4>=4.14.3",
|
||||||
]
|
]
|
||||||
[dependency-groups]
|
|
||||||
dev = [
|
dev = [
|
||||||
"ruff>=0.14.6",
|
"ruff>=0.14.6",
|
||||||
|
"httpx>=0.27.0",
|
||||||
|
"pytest>=8.0.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
authors = [
|
||||||
|
{ name = "Sankalp Tharu", email = "sankalptharu50028@gmail.com" },
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 100
|
||||||
|
target-version = "py313"
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = ["E", "F", "I", "B", "UP", "SIM", "RUF"]
|
||||||
|
ignore = ["E501"]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
aiohttp==3.13.3
|
||||||
|
beautifulsoup4==4.14.3
|
||||||
|
orjson==3.10.18
|
||||||
+1
-4
@@ -1,6 +1,3 @@
|
|||||||
aiohttp==3.13.3
|
|
||||||
beautifulsoup4==4.14.3
|
|
||||||
colorama==0.4.6
|
|
||||||
fastapi==0.128.0
|
fastapi==0.128.0
|
||||||
uvicorn==0.40.0
|
uvicorn==0.40.0
|
||||||
ruff==0.14.6
|
orjson==3.10.18
|
||||||
+267
-151
@@ -1,116 +1,189 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import fcntl
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from contextlib import asynccontextmanager, contextmanager, suppress
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
import orjson
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from colorama import Fore, Style, init
|
|
||||||
|
|
||||||
init(autoreset=True)
|
|
||||||
|
|
||||||
BASE_URL = "https://nepalicalendar.rat32.com/index_nep.php"
|
BASE_URL = "https://nepalicalendar.rat32.com/index_nep.php"
|
||||||
DATA_DIR = "./data"
|
DATA_DIR = Path(os.environ.get("DATA_DIR", "./data")).resolve()
|
||||||
|
|
||||||
|
MIN_YEAR = 1900
|
||||||
|
MAX_YEAR = 2200
|
||||||
|
MAX_MONTH = 12
|
||||||
|
|
||||||
|
REQUEST_TIMEOUT = aiohttp.ClientTimeout(total=30, connect=10)
|
||||||
|
MAX_RETRIES = 3
|
||||||
|
RETRY_BASE_MS = 3000
|
||||||
|
BETWEEN_MONTH_MS = 2000
|
||||||
|
BETWEEN_YEAR_MS = 5000
|
||||||
|
USER_AGENT = (
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger("scraper")
|
||||||
|
|
||||||
|
|
||||||
# Helper: wait between requests
|
def _try_load_existing(file_path: Path) -> dict | None:
|
||||||
async def delay(ms):
|
"""Read file atomically, returning None if it does not exist.
|
||||||
await asyncio.sleep(ms / 1000)
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_month(session, year, month, save_dir=True, attempt=1):
|
|
||||||
month_log = f"[{year}/{month}]"
|
|
||||||
file_path = f"{DATA_DIR}/{year}/{month}.json"
|
|
||||||
|
|
||||||
print(Fore.CYAN + f"\nFetching data for {month_log} (Attempt {attempt})")
|
|
||||||
|
|
||||||
if os.path.exists(file_path):
|
|
||||||
print(Fore.LIGHTBLACK_EX + "Skipped: File already exists.")
|
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
|
||||||
return json.load(f)
|
|
||||||
|
|
||||||
|
Using a single open() with FileNotFoundError closes the classic
|
||||||
|
TOCTOU window where the file could disappear between exists() and open().
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
print(Fore.BLUE + "Sending request...")
|
return orjson.loads(file_path.read_bytes())
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
except (orjson.JSONDecodeError, OSError) as exc:
|
||||||
|
log.warning("Corrupt file %s (%s); re-scraping", file_path, exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_write_json(file_path: Path, payload: dict) -> None:
|
||||||
|
"""Write JSON atomically: temp file in the same directory, then rename.
|
||||||
|
|
||||||
|
os.replace is atomic on POSIX and Windows, so readers (the API) will
|
||||||
|
never see a half-written file even if the process dies mid-write.
|
||||||
|
Writing the temp file in the same directory guarantees rename is
|
||||||
|
on the same filesystem and therefore atomic.
|
||||||
|
"""
|
||||||
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp_name = tempfile.mkstemp(
|
||||||
|
prefix=f".{file_path.name}.",
|
||||||
|
suffix=".tmp",
|
||||||
|
dir=file_path.parent,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "wb") as f:
|
||||||
|
f.write(orjson.dumps(payload, option=orjson.OPT_INDENT_2))
|
||||||
|
os.replace(tmp_name, file_path)
|
||||||
|
except BaseException:
|
||||||
|
with suppress(OSError):
|
||||||
|
os.unlink(tmp_name)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _file_lock(lock_path: Path) -> Iterator[None]:
|
||||||
|
"""Acquire an exclusive OS-level advisory lock on `lock_path`.
|
||||||
|
|
||||||
|
Used to serialise the per-year aggregate write so that two scraper
|
||||||
|
processes cannot read a half-updated set of month files and bake that
|
||||||
|
inconsistency into data/<year>.json. The lock is released on context
|
||||||
|
exit (including exceptions) and the lock file is left on disk as a
|
||||||
|
tiny sentinel so future runs can find it without races of their own.
|
||||||
|
"""
|
||||||
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o644)
|
||||||
|
try:
|
||||||
|
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
with suppress(OSError):
|
||||||
|
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_text(soup: BeautifulSoup, selector: str) -> str:
|
||||||
|
el = soup.select_one(selector)
|
||||||
|
return el.get_text(strip=True) if el else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_list(soup: BeautifulSoup, selector: str, remove_tags: list[str] | None = None) -> list[str]:
|
||||||
|
el = soup.select_one(selector)
|
||||||
|
if el is None:
|
||||||
|
return []
|
||||||
|
for tag in remove_tags or []:
|
||||||
|
child = el.select_one(tag)
|
||||||
|
if child:
|
||||||
|
child.decompose()
|
||||||
|
return [line.strip() for line in el.get_text().split("\n") if line.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_cell(cell_html: str, day_index: int) -> dict:
|
||||||
|
"""Parse one day cell from the calendar grid."""
|
||||||
|
cell = BeautifulSoup(cell_html, "html.parser")
|
||||||
|
n = _extract_text(cell, "#nday")
|
||||||
|
e = _extract_text(cell, "#eday")
|
||||||
|
t = _extract_text(cell, "#dashi")
|
||||||
|
f = _extract_text(cell, "#fest")
|
||||||
|
color = ""
|
||||||
|
font_tag = cell.select_one("#nday font")
|
||||||
|
if font_tag and font_tag.has_attr("color"):
|
||||||
|
color = (font_tag["color"] or "").lower()
|
||||||
|
return {
|
||||||
|
"d": (day_index % 7) + 1,
|
||||||
|
"n": n,
|
||||||
|
"e": e,
|
||||||
|
"t": t,
|
||||||
|
"f": f,
|
||||||
|
"h": color == "red",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_month(
|
||||||
|
session: aiohttp.ClientSession,
|
||||||
|
year: int,
|
||||||
|
month: int,
|
||||||
|
*,
|
||||||
|
save_dir: bool = True,
|
||||||
|
) -> dict | None:
|
||||||
|
"""Fetch a single month's calendar with bounded retries."""
|
||||||
|
if not (MIN_YEAR <= year <= MAX_YEAR) or not 1 <= month <= MAX_MONTH:
|
||||||
|
log.error("Refusing to fetch out-of-range %d/%02d", year, month)
|
||||||
|
return None
|
||||||
|
|
||||||
|
file_path = DATA_DIR / str(year) / f"{month}.json"
|
||||||
|
month_log = f"[{year}/{month:02d}]"
|
||||||
|
|
||||||
|
existing = _try_load_existing(file_path)
|
||||||
|
if existing is not None:
|
||||||
|
log.info("%s Skipped: file already exists", month_log)
|
||||||
|
return existing
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"selYear": year,
|
"selYear": year,
|
||||||
"selMonth": month,
|
"selMonth": month,
|
||||||
"viewCalander": "View Calander",
|
"viewCalander": "View Calander",
|
||||||
}
|
}
|
||||||
|
|
||||||
async with session.post(BASE_URL, data=payload, timeout=15) as resp:
|
last_error: Exception | None = None
|
||||||
|
for attempt in range(1, MAX_RETRIES + 1):
|
||||||
|
log.info("%s Attempt %d/%d", month_log, attempt, MAX_RETRIES)
|
||||||
|
try:
|
||||||
|
async with session.post(BASE_URL, data=payload) as resp:
|
||||||
if resp.status != 200:
|
if resp.status != 200:
|
||||||
raise Exception(f"HTTP {resp.status}")
|
raise aiohttp.ClientResponseError(
|
||||||
|
request_info=resp.request_info,
|
||||||
|
history=resp.history,
|
||||||
|
status=resp.status,
|
||||||
|
)
|
||||||
html = await resp.text()
|
html = await resp.text()
|
||||||
|
|
||||||
print(Fore.GREEN + "Page fetched successfully. Parsing HTML...")
|
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
|
||||||
# Metadata
|
|
||||||
metadata = {
|
metadata = {
|
||||||
"en": soup.select_one("#entarikYr").get_text(strip=True)
|
"en": _extract_text(soup, "#entarikYr"),
|
||||||
if soup.select_one("#entarikYr")
|
"np": _extract_text(soup, "#yren"),
|
||||||
else "",
|
|
||||||
"np": soup.select_one("#yren").get_text(strip=True)
|
|
||||||
if soup.select_one("#yren")
|
|
||||||
else "",
|
|
||||||
}
|
}
|
||||||
print(Fore.LIGHTBLACK_EX + f"Metadata: {metadata['np']} ({metadata['en']})")
|
log.info("%s Metadata: %r (%s)", month_log, metadata["np"], metadata["en"])
|
||||||
|
|
||||||
# Helper to extract lists
|
holi_fest = _extract_list(soup, "#holi", ["b", "a"])
|
||||||
def extract_list(selector, remove_tags=None):
|
marriage = _extract_list(soup, "#bibah", ["b"])
|
||||||
el = soup.select_one(selector)
|
bratabandha = _extract_list(soup, "#bratabandha", ["b"])
|
||||||
if not el:
|
|
||||||
return []
|
|
||||||
remove_tags = remove_tags or []
|
|
||||||
for tag in remove_tags:
|
|
||||||
t = el.select_one(tag)
|
|
||||||
if t:
|
|
||||||
t.decompose()
|
|
||||||
return [line.strip() for line in el.get_text().split("\n") if line.strip()]
|
|
||||||
|
|
||||||
holi_fest = extract_list("#holi", ["b", "a"])
|
days = [
|
||||||
marriage = extract_list("#bibah", ["b"])
|
_parse_cell(str(cell), i)
|
||||||
bratabandha = extract_list("#bratabandha", ["b"])
|
for i, cell in enumerate(soup.select(".cells"))
|
||||||
|
]
|
||||||
print(Fore.MAGENTA + f"Holidays/Festivals: {len(holi_fest)}")
|
|
||||||
print(Fore.MAGENTA + f"Marriage Dates: {len(marriage)}")
|
|
||||||
print(Fore.MAGENTA + f"Bratabandha Dates: {len(bratabandha)}")
|
|
||||||
|
|
||||||
# Extract days
|
|
||||||
days = []
|
|
||||||
cells = soup.select(".cells")
|
|
||||||
for i, cell in enumerate(cells):
|
|
||||||
cell_soup = BeautifulSoup(cell.decode_contents(), "html.parser")
|
|
||||||
|
|
||||||
n = (
|
|
||||||
cell_soup.select_one("#nday").get_text(strip=True)
|
|
||||||
if cell_soup.select_one("#nday")
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
e = (
|
|
||||||
cell_soup.select_one("#eday").get_text(strip=True)
|
|
||||||
if cell_soup.select_one("#eday")
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
t = (
|
|
||||||
cell_soup.select_one("#dashi").get_text(strip=True)
|
|
||||||
if cell_soup.select_one("#dashi")
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
f = (
|
|
||||||
cell_soup.select_one("#fest").get_text(strip=True)
|
|
||||||
if cell_soup.select_one("#fest")
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
color = ""
|
|
||||||
font_tag = cell_soup.select_one("#nday font")
|
|
||||||
if font_tag and font_tag.has_attr("color"):
|
|
||||||
color = font_tag["color"].lower()
|
|
||||||
h = color == "red"
|
|
||||||
|
|
||||||
days.append({"d": (i % 7) + 1, "n": n, "e": e, "t": t, "f": f, "h": h})
|
|
||||||
|
|
||||||
print(Fore.GREEN + f"Parsed {len(days)} days successfully.")
|
|
||||||
|
|
||||||
data = {
|
data = {
|
||||||
"metadata": metadata,
|
"metadata": metadata,
|
||||||
@@ -121,105 +194,148 @@ async def fetch_month(session, year, month, save_dir=True, attempt=1):
|
|||||||
}
|
}
|
||||||
|
|
||||||
if save_dir:
|
if save_dir:
|
||||||
os.makedirs(f"{DATA_DIR}/{year}", exist_ok=True)
|
_atomic_write_json(file_path, data)
|
||||||
with open(file_path, "w", encoding="utf-8") as f:
|
log.info("%s Saved %s (%d days)", month_log, file_path, len(days))
|
||||||
json.dump(
|
|
||||||
data,
|
|
||||||
f,
|
|
||||||
indent=2,
|
|
||||||
ensure_ascii=False,
|
|
||||||
)
|
|
||||||
print(Fore.LIGHTBLACK_EX + f"Saved individual month: {file_path}")
|
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
except (aiohttp.ClientError, TimeoutError, ValueError) as err:
|
||||||
|
last_error = err
|
||||||
|
log.warning("%s Failed: %s", month_log, err)
|
||||||
|
if attempt < MAX_RETRIES:
|
||||||
|
wait_ms = RETRY_BASE_MS * attempt
|
||||||
|
log.info("%s Retrying in %.1fs", month_log, wait_ms / 1000)
|
||||||
|
await asyncio.sleep(wait_ms / 1000)
|
||||||
|
|
||||||
except Exception as err:
|
log.error("%s Giving up after %d attempts: %s", month_log, MAX_RETRIES, last_error)
|
||||||
print(Fore.RED + f"Error fetching {month_log}: {err}")
|
|
||||||
if attempt < 3:
|
|
||||||
wait_time = 3000 * attempt
|
|
||||||
print(Fore.YELLOW + f"Retrying in {wait_time / 1000}s...")
|
|
||||||
await delay(wait_time)
|
|
||||||
return await fetch_month(session, year, month, save_dir, attempt + 1)
|
|
||||||
else:
|
|
||||||
print(Fore.RED + f"Failed after 3 attempts for {month_log}")
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
async def scrape_year(session, year, save_single=True, save_dir=True):
|
@asynccontextmanager
|
||||||
print(Style.BRIGHT + f"\nStarting scrape for year: {Fore.CYAN}{year}")
|
async def _session():
|
||||||
print(
|
connector = aiohttp.TCPConnector(limit=4, ttl_dns_cache=300)
|
||||||
f" Mode: {'Single JSON' if save_single else ''} {'Directory' if save_dir else ''}\n"
|
async with aiohttp.ClientSession(
|
||||||
)
|
timeout=REQUEST_TIMEOUT,
|
||||||
year_data = {}
|
connector=connector,
|
||||||
success_count = 0
|
headers={"User-Agent": USER_AGENT},
|
||||||
|
) as session:
|
||||||
|
yield session
|
||||||
|
|
||||||
for month in range(1, 13):
|
|
||||||
|
async def scrape_year(
|
||||||
|
session: aiohttp.ClientSession,
|
||||||
|
year: int,
|
||||||
|
*,
|
||||||
|
save_single: bool = True,
|
||||||
|
save_dir: bool = True,
|
||||||
|
) -> None:
|
||||||
|
if not (MIN_YEAR <= year <= MAX_YEAR):
|
||||||
|
log.error("Refusing to scrape out-of-range year %d", year)
|
||||||
|
return
|
||||||
|
|
||||||
|
log.info("Starting scrape for year %d", year)
|
||||||
|
year_data: dict[str, dict] = {}
|
||||||
|
success = 0
|
||||||
|
|
||||||
|
for month in range(1, MAX_MONTH + 1):
|
||||||
data = await fetch_month(session, year, month, save_dir=save_dir)
|
data = await fetch_month(session, year, month, save_dir=save_dir)
|
||||||
if data:
|
if data is not None:
|
||||||
year_data[str(month)] = data
|
year_data[str(month)] = data
|
||||||
success_count += 1
|
success += 1
|
||||||
else:
|
else:
|
||||||
print(Fore.RED + f"Failed to gather data for month {month}")
|
log.error("Failed to gather data for %d/%02d", year, month)
|
||||||
|
if month < MAX_MONTH:
|
||||||
|
await asyncio.sleep(BETWEEN_MONTH_MS / 1000)
|
||||||
|
|
||||||
await delay(2000) # delay between months
|
if success == MAX_MONTH and save_single:
|
||||||
|
year_file = DATA_DIR / f"{year}.json"
|
||||||
|
# Serialise with any other scraper touching the same year so we
|
||||||
|
# never read a partial month set and write a mixed aggregate.
|
||||||
|
with _file_lock(DATA_DIR / f"{year}.lock"):
|
||||||
|
_atomic_write_json(year_file, year_data)
|
||||||
|
log.info("Saved aggregated file: %s", year_file)
|
||||||
|
|
||||||
if success_count == 12 and save_single:
|
log.info("Year %d done: success=%d failed=%d", year, success, MAX_MONTH - success)
|
||||||
os.makedirs(DATA_DIR, exist_ok=True)
|
|
||||||
year_file_path = f"{DATA_DIR}/{year}.json"
|
|
||||||
with open(year_file_path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(year_data, f, indent=2, ensure_ascii=False)
|
|
||||||
print(Style.BRIGHT + Fore.GREEN + f"\nSaved aggregated file: {year_file_path}")
|
|
||||||
|
|
||||||
fail_count = 12 - success_count
|
|
||||||
print(Fore.LIGHTBLACK_EX + f"Success: {success_count} Failed: {fail_count}\n")
|
|
||||||
|
|
||||||
|
|
||||||
async def scrape_years(start_year, end_year, save_single=True, save_dir=True):
|
async def scrape_years(
|
||||||
async with aiohttp.ClientSession() as session:
|
start_year: int,
|
||||||
|
end_year: int,
|
||||||
|
*,
|
||||||
|
save_single: bool = True,
|
||||||
|
save_dir: bool = True,
|
||||||
|
) -> None:
|
||||||
|
if start_year > end_year:
|
||||||
|
start_year, end_year = end_year, start_year
|
||||||
|
|
||||||
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
async with _session() as session:
|
||||||
for year in range(start_year, end_year + 1):
|
for year in range(start_year, end_year + 1):
|
||||||
await scrape_year(session, year, save_single, save_dir)
|
await scrape_year(session, year, save_single=save_single, save_dir=save_dir)
|
||||||
if year < end_year:
|
if year < end_year:
|
||||||
print(Fore.YELLOW + "Waiting 5 seconds before next year...")
|
log.info("Waiting %.1fs before next year", BETWEEN_YEAR_MS / 1000)
|
||||||
await delay(5000)
|
await asyncio.sleep(BETWEEN_YEAR_MS / 1000)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
def _parse_args(argv: list[str]) -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(description="Nepali Calendar Scraper")
|
parser = argparse.ArgumentParser(description="Nepali Calendar Scraper")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"years", nargs="+", type=int, help="Year or year range (start end)"
|
"years",
|
||||||
|
nargs="+",
|
||||||
|
type=int,
|
||||||
|
help="One year (e.g. 2082) or a range (e.g. 2070 2085)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--single",
|
"--single",
|
||||||
nargs="?",
|
nargs="?",
|
||||||
const="json",
|
const="json",
|
||||||
help="Generate single JSON (e.g. --single json)",
|
help="Also produce a single aggregated year JSON (data/<year>.json)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--dir",
|
"--dir",
|
||||||
nargs="?",
|
nargs="?",
|
||||||
const="format",
|
const="format",
|
||||||
help="Generate directory format (e.g. --dir format)",
|
help="Also produce per-month files in data/<year>/<month>.json",
|
||||||
)
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Determine output mode
|
def main(argv: list[str] | None = None) -> int:
|
||||||
save_single = False
|
logging.basicConfig(
|
||||||
save_dir = False
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||||
if not args.single and not args.dir:
|
stream=sys.stdout,
|
||||||
# Default behavior: both
|
)
|
||||||
save_single = True
|
args = _parse_args(argv if argv is not None else sys.argv[1:])
|
||||||
save_dir = True
|
|
||||||
else:
|
|
||||||
if args.single:
|
|
||||||
save_single = True
|
|
||||||
if args.dir:
|
|
||||||
save_dir = True
|
|
||||||
|
|
||||||
if len(args.years) == 1:
|
if len(args.years) == 1:
|
||||||
start_year = end_year = args.years[0]
|
start_year = end_year = args.years[0]
|
||||||
|
elif len(args.years) == 2:
|
||||||
|
start_year, end_year = args.years
|
||||||
else:
|
else:
|
||||||
start_year, end_year = args.years[0], args.years[1]
|
log.error("Provide either one year or a start/end pair")
|
||||||
|
return 2
|
||||||
|
|
||||||
asyncio.run(scrape_years(start_year, end_year, save_single, save_dir))
|
if not (MIN_YEAR <= start_year <= MAX_YEAR) or not (MIN_YEAR <= end_year <= MAX_YEAR):
|
||||||
|
log.error("Years must be between %d and %d", MIN_YEAR, MAX_YEAR)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
save_single = bool(args.single)
|
||||||
|
save_dir = bool(args.dir)
|
||||||
|
if not save_single and not save_dir:
|
||||||
|
# Default behaviour matches the original script: produce both.
|
||||||
|
save_single = True
|
||||||
|
save_dir = True
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
scrape_years(
|
||||||
|
start_year,
|
||||||
|
end_year,
|
||||||
|
save_single=save_single,
|
||||||
|
save_dir=save_dir,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
import os
|
|
||||||
import json
|
|
||||||
from fastapi import FastAPI, HTTPException
|
|
||||||
|
|
||||||
app = FastAPI(title="Nepali Calendar API")
|
|
||||||
|
|
||||||
DATA_DIR = "./data"
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/", summary="Root Endpoint")
|
|
||||||
async def root():
|
|
||||||
return {
|
|
||||||
"message": "Welcome to the Nepali Calendar API. Use /docs for interactive API documentation."
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/calendar/{year}", summary="Get Yearly Calendar")
|
|
||||||
async def get_year_calendar(year: int):
|
|
||||||
file_path = f"{DATA_DIR}/{year}.json"
|
|
||||||
if not os.path.exists(file_path):
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=404,
|
|
||||||
detail=f"Calendar data for year {year} not found. Please run the scraper for this year.",
|
|
||||||
)
|
|
||||||
|
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
|
||||||
return json.load(f)
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/calendar/{year}/{month}", summary="Get Monthly Calendar")
|
|
||||||
async def get_month_calendar(year: int, month: int):
|
|
||||||
file_path = f"{DATA_DIR}/{year}/{month}.json"
|
|
||||||
if not os.path.exists(file_path):
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=404,
|
|
||||||
detail=f"Calendar data for {year}/{month} not found. Please run the scraper for this year.",
|
|
||||||
)
|
|
||||||
|
|
||||||
with open(file_path, "r", encoding="utf-8") as f:
|
|
||||||
return json.load(f)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
import uvicorn
|
|
||||||
|
|
||||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
|
||||||
Reference in new issue
Block a user