User-agent: gsfcbot
Disallow: /

User-agent: *
Crawl-delay: 10
Disallow: *.pdf
Disallow: /OpenAPI/
Disallow: /Devel/
Disallow: /temp/
Disallow: /Images/
Disallow: /SCCS/
Disallow: /test/
Disallow: /cgi-bin/
Disallow: /Data/
Disallow: /servlets/
Disallow: /KeywordSearch/
Disallow: /Aboutus/
Disallow: /User/
Disallow: /*/*Portal=GCMD*
Disallow: /portals/Metadata.do*
Disallow: /*sendto_form$
Disallow: /*folder_factories$