#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed

#Penn College
Crawl-delay: 10
Disallow: /student-life/orientation/welcome-weekend/
Disallow: /admissions/scholarships/penn-college-scholarship
Disallow: /node/
Disallow: /r/
Disallow: /catalog/
Disallow: /email/
Disallow: /academics/career-services/career-fair/
Disallow: /academics/career-services/career-fair/directory
Disallow: /academics/career-services/career-fair/register
Disallow: /search
Disallow: /search/
Disallow: /webform/
Disallow: /academicaffairs/
Disallow: /academicsuccess/
Disallow: /admissions_email/
Disallow: /admin/
Disallow: /advertising/
Disallow: /advidement/
Disallow: /armyROTC/
Disallow: /ARP/
Disallow: /artcamp/
Disallow: /askpenncolleg/
Disallow: /ASPIE2016/
Disallow: /assessment/
Disallow: /athletetraining/
Disallow: /athletics/
Disallow: /bcards/
Disallow: /bursar/
Disallow: /busind/
Disallow: /businesscamp/
Disallow: /careerservices/
Disallow: /catalog/majors/GSP.shtml
Disallow: /childcare/
Disallow: /closetohome/
Disallow: /collegehealth/
Disallow: /commencement/
Disallow: /facilitiesandevents/
Disallow: /consumerinfo/
Disallow: /contact/
Disallow: /counseling/
Disallow: /ctm-employee-discount-program
Disallow: /currentstudents/
Disallow: /degrees/
Disallow: /degreesthatwork/
Disallow: /digitalcamp/
Disallow: /dining/
Disallow: /disabilityservices/
Disallow: /distance/
Disallow: /ecards/
Disallow: /emergencyResponse/
Disallow: /esl/
Disallow: /etec/
Disallow: /explore/
Disallow: /finaid/
Disallow: /firstyear/
Disallow: /fitness/
Disallow: /foodcamp/
Disallow: /futurestudents/
Disallow: /giving/
Disallow: /glbt/
Disallow: /governance/
Disallow: /green/
Disallow: /healthcamp/
Disallow: /holidaycard/
Disallow: /homecoming/
Disallow: /homeimages/
Disallow: /hospitalityservices/
Disallow: /humanresources/
Disallow: /ia/
Disallow: /international/
Disallow: /intramurals/
Disallow: /its/
Disallow: /iwanttobe/
Disallow: /jobsatpenncollege/
Disallow: /k12/admin/
Disallow: /k12educators/
Disallow: /library/
Disallow: /nacat/
Disallow: /naturalgas/
Disallow: /newHire/
Disallow: /nssc/
Disallow: /och/
Disallow: /offcampus/
Disallow: /oh2015/
Disallow: /oit/
Disallow: /openhouse/
Disallow: /orientation/
Disallow: /pctoday/
Disallow: /photos/
Disallow: /picnic/
Disallow: /pirc/
Disallow: /police/
Disallow: /president/
Disallow: /privacy/
Disallow: /prm/
Disallow: /profile/
Disallow: /psn/
Disallow: /r/
Disallow: /registrar/
Disallow: /requestinfo/
Disallow: /reslife/
Disallow: /rmce/
Disallow: /scholarships/
Disallow: /schools/
Disallow: /scripts/
Disallow: /share/
Disallow: /sis/
Disallow: /smartgirls/
Disallow: /sonsofitaly/
Disallow: /spring/
Disallow: /studentactivities/
Disallow: /studentaffairs/
Disallow: /studentblog/
Disallow: /studentpolicy/
Disallow: /styles/
Disallow: /sunnyportal/
Disallow: /surplus/
Disallow: /testcenter/
Disallow: /tparticulation/
Disallow: /trail/
Disallow: /transferstudents/
Disallow: /victorianhouse/
Disallow: /videos/
Disallow: /visit/
Disallow: /wdce/
Disallow: /wtc/
Disallow: /files/imported/
Disallow: /admissions/financial-aid/scholarships/penn-college-scholarships
Disallow: /chef
Disallow: /cgi-bin/
Disallow: /files/cgi-bin/

#Bots
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: SemrushBot-SI
Disallow: /