Skip to content
Permalink
Browse files

Patch #624325: urlparse.urlparse() and urlparse.urlsplit() results

now sport attributes that provide access to the parts of the result.
  • Loading branch information...
freddrake committed Apr 1, 2006
1 parent f878b81 commit ad5177cf8da387c203d500d0a65995f9641373ba
Showing with 357 additions and 44 deletions.
  1. +138 −34 Doc/lib/liburlparse.tex
  2. +101 −0 Lib/test/test_urlparse.py
  3. +115 −10 Lib/urlparse.py
  4. +3 −0 Misc/NEWS
@@ -25,48 +25,74 @@ \section{\module{urlparse} ---
\code{nntp}, \code{prospero}, \code{rsync}, \code{rtsp}, \code{rtspu},
\code{sftp}, \code{shttp}, \code{sip}, \code{sips}, \code{snews}, \code{svn},
\code{svn+ssh}, \code{telnet}, \code{wais}.

\versionadded[Support for the \code{sftp} and \code{sips} schemes]{2.5}

The \module{urlparse} module defines the following functions:

\begin{funcdesc}{urlparse}{urlstring\optional{, default_scheme\optional{, allow_fragments}}}
Parse a URL into 6 components, returning a 6-tuple: (addressing
scheme, network location, path, parameters, query, fragment
identifier). This corresponds to the general structure of a URL:
\begin{funcdesc}{urlparse}{urlstring\optional{,
default_scheme\optional{, allow_fragments}}}
Parse a URL into six components, returning a 6-tuple. This
corresponds to the general structure of a URL:
\code{\var{scheme}://\var{netloc}/\var{path};\var{parameters}?\var{query}\#\var{fragment}}.
Each tuple item is a string, possibly empty.
The components are not broken up in smaller parts (e.g. the network
The components are not broken up in smaller parts (for example, the network
location is a single string), and \% escapes are not expanded.
The delimiters as shown above are not part of the tuple items,
The delimiters as shown above are not part of the result,
except for a leading slash in the \var{path} component, which is
retained if present.

Example:

\begin{verbatim}
urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
\end{verbatim}

yields the tuple
retained if present. For example:

\begin{verbatim}
>>> from urlparse import urlparse
>>> o = urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
>>> o
('http', 'www.cwi.nl:80', '/%7Eguido/Python.html', '', '', '')
>>> o.scheme
'http'
>>> o.port
80
>>> o.geturl()
'http://www.cwi.nl:80/%7Eguido/Python.html'
\end{verbatim}

If the \var{default_scheme} argument is specified, it gives the
default addressing scheme, to be used only if the URL string does not
default addressing scheme, to be used only if the URL does not
specify one. The default value for this argument is the empty string.

If the \var{allow_fragments} argument is zero, fragment identifiers
If the \var{allow_fragments} argument is false, fragment identifiers
are not allowed, even if the URL's addressing scheme normally does
support them. The default value for this argument is \code{1}.
support them. The default value for this argument is \constant{True}.

The return value is actually an instance of a subclass of
\pytype{tuple}. This class has the following additional read-only
convenience attributes:

\begin{tableiv}{l|c|l|c}{member}{Attribute}{Index}{Value}{Value if not present}
\lineiv{scheme} {0} {URL scheme specifier} {empty string}
\lineiv{netloc} {1} {Network location part} {empty string}
\lineiv{path} {2} {Hierarchical path} {empty string}
\lineiv{params} {3} {Parameters for last path element} {empty string}
\lineiv{query} {4} {Query component} {empty string}
\lineiv{fragment}{5} {Fragment identifier} {empty string}
\lineiv{username}{ } {User name} {\constant{None}}
\lineiv{password}{ } {Password} {\constant{None}}
\lineiv{hostname}{ } {Host name (lower case)} {\constant{None}}
\lineiv{port} { } {Port number as integer, if present} {\constant{None}}
\end{tableiv}

See section~\ref{urlparse-result-object}, ``Results of
\function{urlparse()} and \function{urlsplit()},'' for more
information on the result object.

\versionchanged[Added attributes to return value]{2.5}
\end{funcdesc}

\begin{funcdesc}{urlunparse}{tuple}
Construct a URL string from a tuple as returned by \code{urlparse()}.
\begin{funcdesc}{urlunparse}{parts}
Construct a URL from a tuple as returned by \code{urlparse()}.
The \var{parts} argument be any six-item iterable.
This may result in a slightly different, but equivalent URL, if the
URL that was parsed originally had redundant delimiters, e.g. a ? with
an empty query (the draft states that these are equivalent).
URL that was parsed originally had unnecessary delimiters (for example,
a ? with an empty query; the RFC states that these are equivalent).
\end{funcdesc}

\begin{funcdesc}{urlsplit}{urlstring\optional{,
@@ -79,12 +105,38 @@ \section{\module{urlparse} ---
separate the path segments and parameters. This function returns a
5-tuple: (addressing scheme, network location, path, query, fragment
identifier).

The return value is actually an instance of a subclass of
\pytype{tuple}. This class has the following additional read-only
convenience attributes:

\begin{tableiv}{l|c|l|c}{member}{Attribute}{Index}{Value}{Value if not present}
\lineiv{scheme} {0} {URL scheme specifier} {empty string}
\lineiv{netloc} {1} {Network location part} {empty string}
\lineiv{path} {2} {Hierarchical path} {empty string}
\lineiv{query} {3} {Query component} {empty string}
\lineiv{fragment} {4} {Fragment identifier} {empty string}
\lineiv{username} { } {User name} {\constant{None}}
\lineiv{password} { } {Password} {\constant{None}}
\lineiv{hostname} { } {Host name (lower case)} {\constant{None}}
\lineiv{port} { } {Port number as integer, if present} {\constant{None}}
\end{tableiv}

See section~\ref{urlparse-result-object}, ``Results of
\function{urlparse()} and \function{urlsplit()},'' for more
information on the result object.

\versionadded{2.2}
\versionchanged[Added attributes to return value]{2.5}
\end{funcdesc}

\begin{funcdesc}{urlunsplit}{tuple}
\begin{funcdesc}{urlunsplit}{parts}
Combine the elements of a tuple as returned by \function{urlsplit()}
into a complete URL as a string.
The \var{parts} argument be any five-item iterable.
This may result in a slightly different, but equivalent URL, if the
URL that was parsed originally had unnecessary delimiters (for example,
a ? with an empty query; the RFC states that these are equivalent).
\versionadded{2.2}
\end{funcdesc}

@@ -93,22 +145,16 @@ \section{\module{urlparse} ---
(\var{base}) with a ``relative URL'' (\var{url}). Informally, this
uses components of the base URL, in particular the addressing scheme,
the network location and (part of) the path, to provide missing
components in the relative URL.

Example:

\begin{verbatim}
urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html')
\end{verbatim}

yields the string
components in the relative URL. For example:

\begin{verbatim}
>>> from urlparse import urljoin
>>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html')
'http://www.cwi.nl/%7Eguido/FAQ.html'
\end{verbatim}

The \var{allow_fragments} argument has the same meaning as for
\code{urlparse()}.
The \var{allow_fragments} argument has the same meaning and default as
for \function{urlparse()}.
\end{funcdesc}

\begin{funcdesc}{urldefrag}{url}
@@ -133,3 +179,61 @@ \section{\module{urlparse} ---
both Uniform Resource Names (URNs) and Uniform Resource
Locators (URLs).}
\end{seealso}


\subsection{Results of \function{urlparse()} and \function{urlsplit()}
\label{urlparse-result-object}}

The result objects from the \function{urlparse()} and
\function{urlsplit()} functions are subclasses of the \pytype{tuple}
type. These subclasses add the attributes described in those
functions, as well as provide an additional method:

\begin{methoddesc}[ParseResult]{geturl}{}
Return the re-combined version of the original URL as a string.
This may differ from the original URL in that the scheme will always
be normalized to lower case and empty components may be dropped.
Specifically, empty parameters, queries, and fragment identifiers
will be removed.

The result of this method is a fixpoint if passed back through the
original parsing function:

\begin{verbatim}
>>> import urlparse
>>> url = 'HTTP://www.Python.org/doc/#'
>>> r1 = urlparse.urlsplit(url)
>>> r1.geturl()
'http://www.Python.org/doc/'
>>> r2 = urlparse.urlsplit(r1.geturl())
>>> r2.geturl()
'http://www.Python.org/doc/'
\end{verbatim}

\versionadded{2.5}
\end{methoddesc}

The following classes provide the implementations of the parse results::

\begin{classdesc*}{BaseResult}
Base class for the concrete result classes. This provides most of
the attribute definitions. It does not provide a \method{geturl()}
method. It is derived from \class{tuple}, but does not override the
\method{__init__()} or \method{__new__()} methods.
\end{classdesc*}


\begin{classdesc}{ParseResult}{scheme, netloc, path, params, query, fragment}
Concrete class for \function{urlparse()} results. The
\method{__new__()} method is overridden to support checking that the
right number of arguments are passed.
\end{classdesc}


\begin{classdesc}{SplitResult}{scheme, netloc, path, query, fragment}
Concrete class for \function{urlsplit()} results. The
\method{__new__()} method is overridden to support checking that the
right number of arguments are passed.
\end{classdesc}
@@ -12,15 +12,53 @@ class UrlParseTestCase(unittest.TestCase):
def checkRoundtrips(self, url, parsed, split):
result = urlparse.urlparse(url)
self.assertEqual(result, parsed)
t = (result.scheme, result.netloc, result.path,
result.params, result.query, result.fragment)
self.assertEqual(t, parsed)
# put it back together and it should be the same
result2 = urlparse.urlunparse(result)
self.assertEqual(result2, url)
self.assertEqual(result2, result.geturl())

# the result of geturl() is a fixpoint; we can always parse it
# again to get the same result:
result3 = urlparse.urlparse(result.geturl())
self.assertEqual(result3.geturl(), result.geturl())
self.assertEqual(result3, result)
self.assertEqual(result3.scheme, result.scheme)
self.assertEqual(result3.netloc, result.netloc)
self.assertEqual(result3.path, result.path)
self.assertEqual(result3.params, result.params)
self.assertEqual(result3.query, result.query)
self.assertEqual(result3.fragment, result.fragment)
self.assertEqual(result3.username, result.username)
self.assertEqual(result3.password, result.password)
self.assertEqual(result3.hostname, result.hostname)
self.assertEqual(result3.port, result.port)

# check the roundtrip using urlsplit() as well
result = urlparse.urlsplit(url)
self.assertEqual(result, split)
t = (result.scheme, result.netloc, result.path,
result.query, result.fragment)
self.assertEqual(t, split)
result2 = urlparse.urlunsplit(result)
self.assertEqual(result2, url)
self.assertEqual(result2, result.geturl())

# check the fixpoint property of re-parsing the result of geturl()
result3 = urlparse.urlsplit(result.geturl())
self.assertEqual(result3.geturl(), result.geturl())
self.assertEqual(result3, result)
self.assertEqual(result3.scheme, result.scheme)
self.assertEqual(result3.netloc, result.netloc)
self.assertEqual(result3.path, result.path)
self.assertEqual(result3.query, result.query)
self.assertEqual(result3.fragment, result.fragment)
self.assertEqual(result3.username, result.username)
self.assertEqual(result3.password, result.password)
self.assertEqual(result3.hostname, result.hostname)
self.assertEqual(result3.port, result.port)

def test_roundtrips(self):
testcases = [
@@ -187,6 +225,69 @@ def test_urldefrag(self):
]:
self.assertEqual(urlparse.urldefrag(url), (defrag, frag))

def test_urlsplit_attributes(self):
url = "HTTP://WWW.PYTHON.ORG/doc/#frag"
p = urlparse.urlsplit(url)
self.assertEqual(p.scheme, "http")
self.assertEqual(p.netloc, "WWW.PYTHON.ORG")
self.assertEqual(p.path, "/doc/")
self.assertEqual(p.query, "")
self.assertEqual(p.fragment, "frag")
self.assertEqual(p.username, None)
self.assertEqual(p.password, None)
self.assertEqual(p.hostname, "www.python.org")
self.assertEqual(p.port, None)
# geturl() won't return exactly the original URL in this case
# since the scheme is always case-normalized
#self.assertEqual(p.geturl(), url)

url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag"
p = urlparse.urlsplit(url)
self.assertEqual(p.scheme, "http")
self.assertEqual(p.netloc, "User:Pass@www.python.org:080")
self.assertEqual(p.path, "/doc/")
self.assertEqual(p.query, "query=yes")
self.assertEqual(p.fragment, "frag")
self.assertEqual(p.username, "User")
self.assertEqual(p.password, "Pass")
self.assertEqual(p.hostname, "www.python.org")
self.assertEqual(p.port, 80)
self.assertEqual(p.geturl(), url)

def test_attributes_bad_port(self):
"""Check handling of non-integer ports."""
p = urlparse.urlsplit("http://www.example.net:foo")
self.assertEqual(p.netloc, "www.example.net:foo")
self.assertRaises(ValueError, lambda: p.port)

p = urlparse.urlparse("http://www.example.net:foo")
self.assertEqual(p.netloc, "www.example.net:foo")
self.assertRaises(ValueError, lambda: p.port)

def test_attributes_without_netloc(self):
# This example is straight from RFC 3261. It looks like it
# should allow the username, hostname, and port to be filled
# in, but doesn't. Since it's a URI and doesn't use the
# scheme://netloc syntax, the netloc and related attributes
# should be left empty.
uri = "sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
p = urlparse.urlsplit(uri)
self.assertEqual(p.netloc, "")
self.assertEqual(p.username, None)
self.assertEqual(p.password, None)
self.assertEqual(p.hostname, None)
self.assertEqual(p.port, None)
self.assertEqual(p.geturl(), uri)

p = urlparse.urlparse(uri)
self.assertEqual(p.netloc, "")
self.assertEqual(p.username, None)
self.assertEqual(p.password, None)
self.assertEqual(p.hostname, None)
self.assertEqual(p.port, None)
self.assertEqual(p.geturl(), uri)


def test_main():
test_support.run_unittest(UrlParseTestCase)

0 comments on commit ad5177c

Please sign in to comment.
You can’t perform that action at this time.