# robots.txt for aniday.com
    
     # Global settings for all crawlers
     User-agent: *
     Disallow: /rss
     Disallow: /chat
     Disallow: /search  # Prevent indexing of internal search results
     Disallow: /admin   # Prevent access to admin areas (if applicable)
     Disallow: /tmp     # Avoid temporary directories being crawled
     Allow: /
    
     # Crawl-delay for better server performance (adjust if needed)
     Crawl-delay: 2
    
     # Specify crawlable file types to improve efficiency
     Allow: /*.js$
     Allow: /*.css$
     Allow: /*.jpg$
     Allow: /*.png$
     Allow: /*.gif$
     Allow: /*.webp$
    
     # Sitemap Index
     Sitemap: https://aniday.com/sitemap_index.xml
    
     # Vietnam-specific Sitemaps
     Sitemap: https://aniday.com/sitemap_vi_landingpages.xml
     Sitemap: https://aniday.com/sitemap_vi_blogs.xml
     Sitemap: https://aniday.com/sitemap_vi_jobs.xml
     Sitemap: https://aniday.com/sitemap_vi_employers.xml
    
     # English-specific Sitemaps
     Sitemap: https://aniday.com/sitemap_en_landingpages.xml
     Sitemap: https://aniday.com/sitemap_en_jobs.xml
     Sitemap: https://aniday.com/sitemap_en_blogs.xml
     Sitemap: https://aniday.com/sitemap_en_headhunters.xml
     Sitemap: https://aniday.com/sitemap_en_employers.xml
    
     # Indonesia-specific Sitemaps
     Sitemap: https://aniday.com/sitemap_id_landingpages.xml
     Sitemap: https://aniday.com/sitemap_id_blogs.xml
    
     # Japan-specific Sitemaps
     Sitemap: https://aniday.com/sitemap_ja_landingpages.xml
     Sitemap: https://aniday.com/sitemap_ja_blogs.xml
    
     # Allow only the main canonical page for categories (if applicable)
     Disallow: /*?sort=
     Disallow: /*?filter=
     Disallow: /*?page=
    
     # Block indexing of duplicate or dynamically generated pages (e.g., session IDs)
     Disallow: /*?session=
    
     # Block sensitive file types (if applicable)
     Disallow: /*.php$
     Disallow: /*.cgi$
     Disallow: /*.asp$
     Disallow: /*.aspx$
    
     # Allow important assets for rich search features
     Allow: /structured-data/
     Allow: /open-graph/
    
     # Enhanced directives for major crawlers
     User-agent: Googlebot
     Disallow: /rss
     Disallow: /chat
     Allow: /
    
     User-agent: Bingbot
     Disallow: /rss
     Disallow: /chat
     Allow: /
    
     User-agent: Yandex
     Disallow: /rss
     Disallow: /chat
     Crawl-delay: 20
    
     # Security and Privacy
     Disallow: /private
     Disallow: /confidential
     Disallow: /backup