# Search engines: the excluded URLs duplicate content on www.gbif.org, index that site instead.
# AI crawlers: GBIF publish monthly snapshots of occurrence data in Parquet format.  Retrieve the data in minutes rather than weeks using the AWS, GCS or Azure-hosted resources listed on https://www.gbif.org/occurrence-snapshots .

User-agent: *
Disallow: *entity=
Disallow: /search
Disallow: /*/search
Disallow: /occurrence
Disallow: /*/occurrence
Disallow: /occurrences
Disallow: /*/occurrences
Disallow: /specimen
Disallow: /*/specimen
Disallow: /specimens
Disallow: /*/specimens
Disallow: /data
Disallow: /*/data
Disallow: /species/search
Disallow: /*/species/search
Disallow: /dataset/search
Disallow: /*/dataset/search
Disallow: /publisher/search
Disallow: /*/publisher/search
Disallow: /grscicoll/collection/search
Disallow: /*/grscicoll/collection/search
Disallow: /grscicoll/institution/search
Disallow: /*/grscicoll/institution/search
Disallow: /grscicoll/person/search
Disallow: /*/grscicoll/person/search
Disallow: /biodiversity-of-bogota
Disallow: /biodiversity-of-canberra
Disallow: /biodiversity-of-oslo
Disallow: /biodiversity-of-porto
Crawl-Delay: 0.333

User-agent: Discordbot
User-agent: LinkedInBot
User-agent: Pinterestbot
User-agent: Slackbot
User-agent: Snapchat
User-agent: TelegramBot
User-agent: Twitterbot
User-agent: WhatsApp
User-agent: facebookexternalhit
Disallow: