# Sayulita.com
#
# The previous version of this file said:
#
#     User-agent: *
#     Allow: /application/vies/pages/
#
# which does nothing useful twice over. "Allow" only carves an exception out of
# a "Disallow", and there was none, so everything was crawlable already; and the
# path is a misspelling of application/views, a directory that should not be
# reachable in the first place. What it did carry that mattered was the Sitemap
# line, pointing at a static file last edited in August 2021 -- fifty-seven of
# whose hundred and thirty-seven addresses had since gone. That file is now
# generated from the site's own data.

User-agent: *

# Editing, sign-in and the upload endpoints. Nothing a reader wants, and a
# crawler following them just fills the log with redirects to the login page.
Disallow: /admin
Disallow: /cms/
Disallow: /users/
Disallow: /editor
Disallow: /posts/create
Disallow: /posts/update
Disallow: /posts/search
Disallow: /posts/upload_cover
Disallow: /categories/create
Disallow: /categories/edit/
Disallow: /categories/update

# /catalog/<anything> answers 200 with an empty listing rather than a 404, so a
# crawler cannot tell a real category from a typo and will index either.
Disallow: /catalog/

# Crawls hard enough to show up in the logs.
User-agent: SemrushBot
Crawl-delay: 60

# The canonical host: .htaccess redirects http and the bare domain to this one,
# so naming it here saves every crawler a hop.
Sitemap: https://www.sayulita.com/sitemap.xml