# robots.txt — unrealestate.in (Unreal Living)
#
# Vite copies everything in frontend/public/ to dist/ verbatim, so this file is
# served at https://unrealestate.in/robots.txt with no nginx change needed.
#
# Two jobs here:
#   1. Keep crawlers out of the parts of the site that are private, personalised
#      or transactional. Those pages can never rank — they need a session — and
#      every one a crawler fetches is crawl budget not spent on a property page.
#   2. Explicitly welcome the AI/answer-engine crawlers. Travel discovery is
#      moving into ChatGPT, Perplexity and AI Overviews, and those crawlers only
#      index what they are permitted to fetch. Silence is not consent to them in
#      the way it is to Googlebot — several default to conservative behaviour.

# ── Default policy for every crawler ─────────────────────────────────────────
User-agent: *

# Admin panel. Login-walled, zero search value, and we would rather it not be
# enumerated in an index at all.
Disallow: /admin
Disallow: /admin/

# Transactional funnel. These carry guest details and the Razorpay handoff, and
# they are meaningless without a booking in progress.
Disallow: /book
Disallow: /booking/
Disallow: /review/

# Guest account area — personalised, session-gated.
Disallow: /login
Disallow: /signup
Disallow: /profile
Disallow: /my-bookings
Disallow: /favourites

# The API is not a set of web pages. Blocking it stops crawlers burning budget on
# JSON endpoints that would otherwise be discovered through the network tab of a
# rendering crawler.
Disallow: /api/

# Filter/sort permutations on the listing page are near-infinite and all resolve
# to the same inventory. Canonical tags handle the duplication for engines that
# render; this stops the crawl waste for those that do not. The bare /properties
# page stays fully crawlable.
Disallow: /properties?*
Disallow: /*?checkin=
Disallow: /*?checkout=
Disallow: /*?guests=

# Everything else — home, city pages, property pages, content pages — is open.
Allow: /

# ── AI / answer-engine crawlers ──────────────────────────────────────────────
# Deliberately allowed. These are read-for-answers crawlers, and being absent
# from them means being absent from "where should I stay in Goa" asked to an
# assistant. They inherit the Disallow list above by repeating it, because a
# named User-agent block replaces the * block rather than adding to it.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bingbot
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
Disallow: /admin
Disallow: /admin/
Disallow: /book
Disallow: /booking/
Disallow: /review/
Disallow: /login
Disallow: /signup
Disallow: /profile
Disallow: /my-bookings
Disallow: /favourites
Disallow: /api/
Allow: /

# ── Crawlers we do not want ──────────────────────────────────────────────────
# SEO backlink scrapers. They provide us nothing, they are aggressive, and they
# hand our inventory and pricing structure to competitors doing site audits.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: PetalBot
Disallow: /

Sitemap: https://unrealestate.in/sitemap.xml