Cursor and Pagination

# Cursor fundamentals — lazy evaluation and batch fetching

from pymongo import MongoClient

client = MongoClient("mongodb://localhost:27017/")
db     = client["dataplexa"]

# find() returns a cursor immediately — no network round trip yet
cursor = db.products.find({})
print("Type of cursor:", type(cursor).__name__)
print("Query not executed yet — cursor is lazy")

# Iteration triggers the actual query execution
print("\nIterating cursor — query executes now:")
for product in cursor:
    print(f"  {product['name']}")

# A cursor can only be iterated once
print("\nTrying to iterate again:")
second_pass = list(cursor)   # cursor is exhausted
print(f"  Documents from second pass: {len(second_pass)}")
print("  Cursor exhausted — re-run the query to iterate again")

# Re-run the query for a fresh cursor
fresh_cursor = db.products.find({}, {"name": 1, "_id": 0})
all_products = list(fresh_cursor)
print(f"\nFresh cursor — {len(all_products)} products loaded into memory")

# Cursor methods — chaining before execution

from pymongo import MongoClient, ASCENDING, DESCENDING

client = MongoClient("mongodb://localhost:27017/")
db     = client["dataplexa"]

# Full method chain — none execute until iteration begins
cursor = (
    db.products
    .find({"category": "Electronics"}, {"name": 1, "price": 1, "_id": 0})
    .sort("price", DESCENDING)
    .skip(1)
    .limit(2)
)

print("Top 2nd and 3rd most expensive Electronics:")
for p in cursor:
    print(f"  ${p['price']:>7.2f}  {p['name']}")

# Cursor introspection — check before iterating
cursor2 = db.orders.find({"status": "delivered"})
print(f"\nEstimated documents in cursor: not available until iterated")
print(f"Count via collection method: {db.orders.count_documents({'status': 'delivered'})}")

# batch_size — control how many documents per network round trip
large_cursor = db.products.find({}).batch_size(3)
print("\nIterating with batch_size=3:")
for p in large_cursor:
    print(f"  {p['name']}")

# Cursor timeout — understanding expiry and the no_cursor_timeout option

from pymongo import MongoClient

client = MongoClient("mongodb://localhost:27017/")
db     = client["dataplexa"]

# Default cursor — times out after 10 minutes of inactivity
normal_cursor = db.products.find({})

# no_cursor_timeout=True — server keeps cursor alive indefinitely
# Use only when necessary — always close the cursor explicitly
persistent_cursor = db.products.find({}, no_cursor_timeout=True)

try:
    for product in persistent_cursor:
        # Simulate slow processing
        print(f"  Processing: {product['name']}")
        # ... expensive operation here ...
finally:
    # Always close a no_cursor_timeout cursor — otherwise it leaks on the server
    persistent_cursor.close()
    print("Cursor closed — server resources released")

# Context manager — automatically closes the cursor on exit
print("\nUsing cursor as context manager:")
with db.products.find({}, {"name": 1, "_id": 0}) as ctx_cursor:
    for p in ctx_cursor:
        print(f"  {p['name']}")
# cursor closed automatically here

# Offset-based pagination — skip() + limit()

from pymongo import MongoClient, ASCENDING
import math

client = MongoClient("mongodb://localhost:27017/")
db     = client["dataplexa"]

PAGE_SIZE  = 3
total_docs = db.products.count_documents({})
total_pages = math.ceil(total_docs / PAGE_SIZE)

print(f"Total products: {total_docs}  |  Page size: {PAGE_SIZE}  |  Total pages: {total_pages}")

def get_page_offset(page: int, page_size: int = PAGE_SIZE):
    """Fetch one page using skip/limit offset pagination."""
    skip_n = (page - 1) * page_size
    docs   = list(
        db.products
        .find({}, {"name": 1, "price": 1, "_id": 0})
        .sort("price", ASCENDING)
        .skip(skip_n)
        .limit(page_size)
    )
    return {
        "page":       page,
        "total_pages": total_pages,
        "results":    docs,
        "has_next":   page < total_pages,
        "has_prev":   page > 1,
    }

for page_num in range(1, total_pages + 1):
    page_data = get_page_offset(page_num)
    print(f"\nPage {page_data['page']} of {page_data['total_pages']}  "
          f"[prev:{page_data['has_prev']} next:{page_data['has_next']}]")
    for p in page_data["results"]:
        print(f"  ${p['price']:>7.2f}  {p['name']}")

# Keyset pagination — O(1) per page using a range filter anchor

from pymongo import MongoClient, ASCENDING

client = MongoClient("mongodb://localhost:27017/")
db     = client["dataplexa"]

PAGE_SIZE = 3

def get_page_keyset(last_price=None, last_id=None, page_size=PAGE_SIZE):
    """
    Fetch the next page of products sorted by price ascending.
    Pass price and _id of the last document from the previous page.
    Returns: list of documents and the cursor values for the next page.
    """
    if last_price is None:
        # First page — no anchor
        filter_q = {}
    else:
        # Subsequent pages — anchor after the last seen document
        # Tie-break on _id ensures stability when multiple docs share the same price
        filter_q = {"$or": [
            {"price": {"$gt": last_price}},
            {"price": last_price, "_id": {"$gt": last_id}}
        ]}

    docs = list(
        db.products
        .find(filter_q, {"name": 1, "price": 1, "_id": 1})
        .sort([("price", ASCENDING), ("_id", ASCENDING)])
        .limit(page_size)
    )

    next_anchor = {"last_price": docs[-1]["price"], "last_id": docs[-1]["_id"]} if docs else None
    return docs, next_anchor

# Page 1
page, anchor = get_page_keyset()
print("Page 1 (keyset):")
for p in page:
    print(f"  ${p['price']:>7.2f}  {p['name']}")

# Page 2 — pass anchor from page 1
page, anchor = get_page_keyset(**anchor)
print("\nPage 2 (keyset):")
for p in page:
    print(f"  ${p['price']:>7.2f}  {p['name']}")

# Page 3 — pass anchor from page 2
page, anchor = get_page_keyset(**anchor)
print("\nPage 3 (keyset):")
for p in page:
    print(f"  ${p['price']:>7.2f}  {p['name']}")

print("\nNo more pages" if anchor and db.products.count_documents(
    {"$or": [{"price": {"$gt": anchor["last_price"]}},
              {"price": anchor["last_price"], "_id": {"$gt": anchor["last_id"]}}]}
) == 0 else "More pages may exist")

# Offset vs keyset — choosing the right pagination strategy

comparison = {
    "Feature": [
        "Implementation complexity",
        "Performance at page 1",
        "Performance at page 1000",
        "Random page access (jump to page N)",
        "Stable results during concurrent writes",
        "Best for",
    ],
    "Offset (skip + limit)": [
        "Simple",
        "Fast",
        "Slow — scans all skipped docs",
        "Yes — trivial",
        "No — inserts shift page boundaries",
        "Admin UIs, small datasets, known total pages",
    ],
    "Keyset (range filter)": [
        "Moderate",
        "Fast",
        "Fast — always O(1) index scan",
        "No — forward only",
        "Yes — anchor is stable",
        "APIs, infinite scroll, large datasets",
    ],
}

header = f"{'Feature':40} {'Offset':35} {'Keyset':35}"
print(header)
print("-" * len(header))
for i, feature in enumerate(comparison["Feature"]):
    offset = comparison["Offset (skip + limit)"][i]
    keyset = comparison["Keyset (range filter)"][i]
    print(f"{feature:40} {offset:35} {keyset:35}")