User-agent: *
Disallow: /cgi-bin/	  # scripts area
Disallow: /js/		  # Javascript
Disallow: /css/           # Style Sheets

# TJK 9/5/2019 - allow these as per Bobby's email (disabled below for
# specific crawlers
#Disallow: /pub/data/
#Disallow: /data/
#Disallow: /cdaweb/istp_public/
#Disallow: /istp_public/
#Disallow: /cdaweb/pre_istp/
#Disallow: /pre_istp/
#Disallow: /cdaweb/lunar/
#Disallow: /lunar/
#Disallow: /sp_phys/
Disallow: /WS/                         # CDAS Web Services
#
Disallow: /cgi-bin                      # JWJ per task request 3037
Disallow: /registry/vitmo/query.xql    # VITMO RESTful Web Services
Disallow: /opendap/
Disallow: /include                  # server-side include files
Disallow: /stats                    # statistics files
Disallow: /style                    # cascading style sheets
Disallow: /logo                     # various graphics files
#
User-agent: googlebot
Disallow: /pub/data/
Disallow: /data/
Disallow: /cdaweb/istp_public/
Disallow: /istp_public/
Disallow: /cdaweb/pre_istp/
Disallow: /pre_istp/
Disallow: /sp_phys/
Disallow: /WS/

User-agent: bingbot
Disallow: /pub/data/
Disallow: /data/
Disallow: /cdaweb/istp_public/
Disallow: /istp_public/
Disallow: /cdaweb/pre_istp/
Disallow: /pre_istp/
Disallow: /sp_phys/
Disallow: /WS/

User-agent: sogou
Disallow: /pub/data/
Disallow: /data/
Disallow: /cdaweb/istp_public/
Disallow: /istp_public/
Disallow: /cdaweb/pre_istp/
Disallow: /pre_istp/
Disallow: /sp_phys/
Disallow: /WS/

User-agent: baiduspider
Disallow: /pub/data/
Disallow: /data/
Disallow: /cdaweb/istp_public/
Disallow: /istp_public/
Disallow: /cdaweb/pre_istp/
Disallow: /pre_istp/
Disallow: /sp_phys/
Disallow: /WS/

User-agent: yandex
Disallow: /

User-agent: slurp
Disallow: /pub/data/
Disallow: /data/
Disallow: /cdaweb/istp_public/
Disallow: /istp_public/
Disallow: /cdaweb/pre_istp/
Disallow: /pre_istp/
Disallow: /sp_phys/
Disallow: /WS/

User-agent: exabot
Disallow: /pub/data/
Disallow: /data/
Disallow: /cdaweb/istp_public/
Disallow: /istp_public/
Disallow: /cdaweb/pre_istp/
Disallow: /pre_istp/
Disallow: /sp_phys/
Disallow: /WS/

User-agent: facebot
Disallow: /pub/data/
Disallow: /data/
Disallow: /cdaweb/istp_public/
Disallow: /istp_public/
Disallow: /cdaweb/pre_istp/
Disallow: /pre_istp/
Disallow: /sp_phys/
Disallow: /WS/

User-agent: PiplBot
Disallow: /