| Current File : //opt/imh-python/lib/python2.7/test/test_robotparser.pyo |
�
��Wc @ s� d d l Z d d l Z d d l Z d d l m Z d d l m Z m Z e Z y d d l m
Z
Wn e k
r} e Z n Xd e j
f d � � YZ e j � Z d d � Z d Z d
d g Z d d
d g Z e d e e e � d Z d
d dG g Z d g Z e d e e e � d Z g Z d d
d g Z e d e e e � d Z g Z d d d d d d d g Z e d e e e d � e d e e e d! � d" Z d g Z d d d d d# d$ g Z e d% e e e � d& Z d g Z g Z e d' e e e � d( Z d) g Z d* g Z e d+ e e e d, d- �d. Z g Z d/ g Z e d0 e e e d, d- �g Z d/ g Z e d1 e e e d, d2 �d3 Z g Z d/ g Z e d4 e e e d, d- �d/ g Z g Z e d5 e e e d, d2 �d( Z d) g Z d* g Z e d6 e e e d, d7 �d8 Z d9 g Z d: g Z e d; e e e � d< Z d= g Z d9 g Z e d> e e e � d? Z d@ g Z dA g Z e dB e e e � dC e j
f dD � � YZ dE � Z e dF k r�d e _ e � n d S(H i����N( t test_support( t urlopent HTTPError( t HTTPSHandlert
RobotTestCasec B s# e Z d � Z d � Z d � Z RS( c C sg t j j | � | r, d | | f | _ n d | | f | _ | | _ | | _ | | _ | | _ d S( Ns RobotTest(%d, good, %s)s RobotTest(%d, bad, %s)( t unittestt TestCaset __init__t strt parsert urlt goodt agent( t selft indexR R
R R ( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyR s c C s~ t | j t � r$ | j \ } } n | j } | j } | j r^ | j | j j | | � � n | j | j j | | � � d S( N( t
isinstanceR
t tupleR R t
assertTrueR t can_fetcht assertFalse( R
R R
( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyt runTest s c C s | j S( N( R ( R
( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyt __str__" s ( t __name__t
__module__R R R ( ( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyR s t test_robotparserc C s� t j | � j � } t j � } | j | � x- | D]% } t j t | | | d | � � q5 Wx- | D]% } t j t | | | d | � � qe Wd S( Ni i ( t StringIOt readlinest robotparsert RobotFileParsert parset testst addTestR ( R t
robots_txtt good_urlst bad_urlsR t linesR R
( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyt RobotTest' s
#
s�
User-agent: *
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
Disallow: /tmp/ # these will soon disappear
Disallow: /foo.html
t /s
/test.htmls /cyberworld/map/index.htmls /tmp/xxxs /foo.htmli s�
# robots.txt for http://www.example.com/
User-agent: *
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow:
t cybermapperi s%
# go away
User-agent: *
Disallow: /
s /tmp/i sm
User-agent: figtree
Disallow: /tmp
Disallow: /a%3cd.html
Disallow: /a%2fb.html
Disallow: /%7ejoe/index.html
s /tmps /tmp.htmls /tmp/a.htmls /a%3cd.htmls /a%3Cd.htmls /a%2fb.htmls /~joe/index.htmli t figtreei s FigTree Robot libwww-perl/5.04sf
User-agent: *
Disallow: /tmp/
Disallow: /a%3Cd.html
Disallow: /a/b.html
Disallow: /%7ejoe/index.html
s /a/b.htmls /%7Ejoe/index.htmli s
User-Agent: *
Disallow: /.
i sG
User-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
s /folder1/myfile.htmls /folder1/anotherfile.htmli R t GooglebotsJ
User-agent: Googlebot
Disallow: /
User-agent: Googlebot-Mobile
Allow: /
s /something.jpgi i
s Googlebot-MobilesJ
User-agent: Googlebot-Mobile
Allow: /
User-agent: Googlebot
Disallow: /
i i i
t googlebots/
User-agent: *
Disallow: /some/path?name=value
s
/some/paths /some/path?name=valuei sK
User-agent: *
Disallow: /some/path
User-agent: *
Disallow: /another/path
s
/another/pathi s;
User-agent: *
Allow: /some/path?
Disallow: /another/path?
s /some/path?s /another/path?i t NetworkTestCasec B s5 e Z d � Z e j e d � e j d � � � Z RS( c C s t j d � t j d � �� d } | d } y t | � WnE t k
r� } | j d d h k r� | j d | | j f � q� n X| j d | � t j � } | j | � y | j
� Wn" t k
r� | j d | � n X| j | j
d
| � t � Wd QXd S( Nt networks mueblesmoraleda.coms http://mueblesmoraleda.coms /robots.txti� i� s0 %r should return a 401 or 403 HTTP error, not %rs5 %r should return a 401 or 403 HTTP error, not succeeds %s is unavailablet *( R t requirest transient_internetR R t codet skipTestR R t set_urlt readt IOErrort assertEqualR t False( R
R
t
robots_urlt eR ( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyt testPasswordProtectedSite� s*
s$ need SSL support to download licensec C sY t j d � t j d � �7 t j d � } | j � | j | j d d � � Wd QXd S( NR+ s www.python.orgs! https://www.python.org/robots.txtR, ( R R- R. R R R2 R R ( R
R ( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyt
testPythonOrg s
( R R R8 R t
skipUnlesst
HAVE_HTTPSR t system_must_validate_certR9 ( ( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyR* � s c C s t j t � t j t � d S( N( R t run_unittestR R* ( ( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyt test_main" s
t __main__( R& s /cyberworld/map/index.html( R R R t testR t urllib2R R t TrueR; R t ImportErrorR5 R R t TestSuiteR R$ t docR t badR* R> R t verbose( ( ( sn /builddir/build/BUILDROOT/imh-python-2.7.12-1.el7.x86_64/opt/imh-python/lib/python2.7/test/test_robotparser.pyt <module> s� $
(