robots.txt

Deploy this file at https://pesticideccb.org/robots.txt. The sitemap URL references the auto-updating sitemap function.

# robots.txt — pesticideccb.org
# Last updated: 2026-03-01

# ════════════════════════════════════════════════════════════════════════════
# All crawlers — default rules
# ════════════════════════════════════════════════════════════════════════════
User-agent: *

# ── Explicitly allow all high-value service & content pages ────────────────
Allow: /
Allow: /certify-product
Allow: /amazon-fast-track
Allow: /verify-certification
Allow: /contact
Allow: /manufacturers
Allow: /distributors-wholesalers
Allow: /retailers-marketplaces
Allow: /regulatory-compliance
Allow: /importers-exporters
Allow: /r-d-organizations
Allow: /brand-owners
Allow: /regulatory-strategy
Allow: /product-registration
Allow: /label-and-claims
Allow: /data-and-studies
Allow: /agency-submissions
Allow: /compliance-audits
Allow: /due-diligence
Allow: /training-programs
Allow: /fifra-guidelines
Allow: /epa-regulations
Allow: /regulatory-glossary
Allow: /faqs
Allow: /state-resources
Allow: /industry-insights
Allow: /regulatory-updates
Allow: /about
Allow: /careers
Allow: /privacy-policy
Allow: /terms-of-use
Allow: /accessibility-statement
Allow: /conflicts-of-interest

# ── Allow all static assets (CSS, JS, fonts, images) ──────────────────────
Allow: /*.css$
Allow: /*.js$
Allow: /*.woff2$
Allow: /*.woff$
Allow: /*.ttf$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.ico$

# ── Duplicate / alias pages ────────────────────────────────────────────────
Disallow: /home

# ── Auth-gated portal & application pages ──────────────────────────────────
Disallow: /client-portal
Disallow: /client-dashboard
Disallow: /certification-portal
Disallow: /company-onboarding
Disallow: /product-certification-application
Disallow: /application-detail
Disallow: /application-documents
Disallow: /certification-success
Disallow: /notification-preferences

# ── Admin / internal management pages ──────────────────────────────────────
Disallow: /admin-dashboard
Disallow: /admin-application-review
Disallow: /admin-renewal-management
Disallow: /admin-audit-log
Disallow: /admin-setup
Disallow: /manage-users
Disallow: /manage-api-keys
Disallow: /billing

# ── Thin / low-value / internal content pages ──────────────────────────────
Disallow: /entomology-support
Disallow: /compliance-resources
Disallow: /compliance-updates
Disallow: /compliance-checker
Disallow: /distributor-compliance
Disallow: /additional-services
Disallow: /premium-services
Disallow: /annual-renewals
Disallow: /agricultural-pesticides
Disallow: /pesticide-devices
Disallow: /pesticide-products
Disallow: /training-representation
Disallow: /subscribe-newsletter
Disallow: /news
Disallow: /case-studies

# ── Dev / internal tool pages ──────────────────────────────────────────────
Disallow: /seo-dashboard
Disallow: /upload-animated-logo
Disallow: /upload-logo
Disallow: /upload-web-m
Disallow: /update-certification
Disallow: /logo-options
Disallow: /sitemap-xml
Disallow: /robots

# ── Query-string / parameter URLs (search results, filters, duplicates) ────
Disallow: /*?

# ── Crawl-delay ────────────────────────────────────────────────────────────
Crawl-delay: 1

# ════════════════════════════════════════════════════════════════════════════
# Googlebot — no crawl-delay needed (Google manages its own rate)
# ════════════════════════════════════════════════════════════════════════════
User-agent: Googlebot
Allow: /
Allow: /*.css$
Allow: /*.js$
Disallow: /home
Disallow: /client-portal
Disallow: /client-dashboard
Disallow: /certification-portal
Disallow: /company-onboarding
Disallow: /product-certification-application
Disallow: /application-detail
Disallow: /application-documents
Disallow: /certification-success
Disallow: /notification-preferences
Disallow: /admin-dashboard
Disallow: /admin-application-review
Disallow: /admin-renewal-management
Disallow: /admin-audit-log
Disallow: /admin-setup
Disallow: /manage-users
Disallow: /manage-api-keys
Disallow: /billing
Disallow: /seo-dashboard
Disallow: /upload-animated-logo
Disallow: /upload-logo
Disallow: /upload-web-m
Disallow: /update-certification
Disallow: /logo-options
Disallow: /sitemap-xml
Disallow: /robots
Disallow: /*?

# ════════════════════════════════════════════════════════════════════════════
# Sitemap
# ════════════════════════════════════════════════════════════════════════════
Sitemap: https://pesticideccb.org/api/functions/generateSitemap