# Karan Gupta Consulting - robots.txt # AI-Friendly & SEO Optimized # =========================================== # WELCOME ALL SEARCH ENGINE CRAWLERS # =========================================== User-agent: * Allow: / # Critical Content Pillars Allow: /universities/ Allow: /study-abroad/ Allow: /services/ Allow: /admissions-strategy/ Allow: /success-stories/ Allow: /blog/ Allow: /reels/ Allow: /scholarships Allow: /media Allow: /test-preparation/ Allow: /learn/ Allow: /about Allow: /contact Allow: /free-course Allow: /get-started # =========================================== # GOOGLE BOTS - FULL ACCESS # =========================================== User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Video Allow: / User-agent: Googlebot-News Allow: / User-agent: Google-Extended Allow: / User-agent: Storebot-Google Allow: / User-agent: GoogleOther Allow: / # =========================================== # AI & LLM TRAINING BOTS - FULL ACCESS # =========================================== User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: CCBot Allow: / User-agent: anthropic-ai Allow: / User-agent: Claude-Web Allow: / User-agent: Claudebot Allow: / User-agent: cohere-ai Allow: / User-agent: PerplexityBot Allow: / User-agent: Bytespider Allow: / User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / User-agent: meta-externalagent Allow: / User-agent: FacebookBot Allow: / User-agent: facebookexternalhit Allow: / User-agent: Amazonbot Allow: / User-agent: YouBot Allow: / User-agent: Diffbot Allow: / User-agent: omgili Allow: / User-agent: omgilibot Allow: / # =========================================== # OTHER SEARCH ENGINES - FULL ACCESS # =========================================== User-agent: Bingbot Allow: / User-agent: BingPreview Allow: / User-agent: msnbot Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: YandexBot Allow: / User-agent: Sogou Allow: / User-agent: Exabot Allow: / User-agent: ia_archiver Allow: / # =========================================== # SOCIAL MEDIA CRAWLERS - FULL ACCESS # =========================================== User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Pinterest Allow: / User-agent: Pinterestbot Allow: / User-agent: Slackbot Allow: / User-agent: WhatsApp Allow: / User-agent: TelegramBot Allow: / User-agent: Discordbot Allow: / User-agent: Embedly Allow: / User-agent: Quora-Bot Allow: / User-agent: redditbot Allow: / # =========================================== # SEO & ANALYTICS TOOLS - FULL ACCESS # =========================================== User-agent: SemrushBot Allow: / User-agent: AhrefsBot Allow: / User-agent: MJ12bot Allow: / User-agent: DotBot Allow: / User-agent: Screaming Frog SEO Spider Allow: / User-agent: rogerbot Allow: / User-agent: SiteAuditBot Allow: / User-agent: Chrome-Lighthouse Allow: / User-agent: JEGL Allow: / # =========================================== # PRERENDER & CACHING BOTS # =========================================== User-agent: Prerender Allow: / User-agent: prerender Allow: / # =========================================== # RESTRICTED AREAS (All Bots) # =========================================== User-agent: * Disallow: /admin/ Disallow: /portal/ Disallow: /staff/ Disallow: /api/auth/ Disallow: /api/admin/ Disallow: /login Disallow: /register Disallow: /forgot-password Disallow: /reset-password Disallow: /search Disallow: /search? # Free Course - Only landing page is public, rest is behind login Disallow: /free-course/thank-you Disallow: /free-course/certificate Disallow: /free-course/day/ # Psychometric tests - Behind login Disallow: /psychometric-test/ # Block legacy URLs that no longer exist Disallow: /upload/ Disallow: /resize.php Disallow: /forgotpassword/ Disallow: /kcfinder/ # Block broken /watch/ URLs (legacy video pages) Disallow: /watch/ # Block broken /reels/ URLs with slugs (legacy reel pages) Disallow: /reels/comment- Disallow: /reels/heres- Disallow: /reels/ivy-league- Disallow: /reels/study-abroad- Disallow: /reels/in-the-us- Disallow: /reels/watch- Disallow: /reels/while- Disallow: /reels/whats- Disallow: /reels/for-some- Disallow: /reels/is-pcmb- Disallow: /reels/are-your- Disallow: /reels/there-are- Disallow: /reels/from-maggie- Disallow: /reels/finally- Disallow: /reels/very-soon- Disallow: /reels/reel- Disallow: /reels/18 Disallow: /reels/17 Disallow: /reels/think- Disallow: /reels/doctor- Disallow: /reels/stop- Disallow: /reels/tea-or- Disallow: /reels/what- Disallow: /reels/your- Disallow: /reels/lets- Disallow: /reels/is-the- Disallow: /reels/as-the- Disallow: /reels/studying- Disallow: /reels/internships- Disallow: /reels/international- Disallow: /reels/want-to- Disallow: /reels/uk-scholarship- Disallow: /reels/sat-prep- Disallow: /reels/study-in- Disallow: /reels/non-russell- Disallow: /reels/curriculum- Disallow: /reels/ib-study- Disallow: /reels/scholarship- Disallow: /reels/mba- Disallow: /reels/ms-vs- Disallow: /reels/expensive- Disallow: /reels/fashion- Disallow: /reels/college-list- Disallow: /reels/top-paying- Disallow: /reels/uk-visa- Disallow: /reels/europe- # Block legacy test pages Disallow: /test/welcome/ # Block legacy blog URLs with numeric IDs Disallow: /blog/category/ Disallow: /gyan/article/1 Disallow: /gyan/article/3 Disallow: /gyan/article/5 Disallow: /gyan/article/6 Disallow: /gyan/article/7 Disallow: /gyan/article/8 Disallow: /gyan/article/10 # Block query parameter variations (prevent duplicate content indexing) Disallow: /*?no_redirect= Disallow: /*?from=country Disallow: /*?C= Disallow: /*?O= Disallow: /*?platform= Disallow: /*?utm_source= Disallow: /*?path= Disallow: /*?search= Disallow: /blog?tag= Disallow: /blog?category= Disallow: /blog?page= Disallow: /blog?q= Disallow: /blog?search= Disallow: /reels?tag= Disallow: /reels?category= Disallow: /testimonials?page= Disallow: /testimonials?no_redirect= Disallow: /student-reviews?page= Disallow: /student-reviews?no_redirect= Disallow: /media?page= Disallow: /media?no_redirect= Disallow: /success-stories?page= Disallow: /student-outcomes?country= # Block development/debug files Disallow: /*.json$ Disallow: /static/js/*.map # =========================================== # SITEMAP LOCATION # =========================================== Sitemap: https://www.karangupta.com/sitemap.xml