retry.py revision 420216e5
1420216e5SHanoh Haimfrom __future__ import absolute_import
2420216e5SHanoh Haimimport time
3420216e5SHanoh Haimimport logging
4420216e5SHanoh Haimfrom collections import namedtuple
5420216e5SHanoh Haimfrom itertools import takewhile
6420216e5SHanoh Haimimport email
7420216e5SHanoh Haimimport re
8420216e5SHanoh Haim
9420216e5SHanoh Haimfrom ..exceptions import (
10420216e5SHanoh Haim    ConnectTimeoutError,
11420216e5SHanoh Haim    MaxRetryError,
12420216e5SHanoh Haim    ProtocolError,
13420216e5SHanoh Haim    ReadTimeoutError,
14420216e5SHanoh Haim    ResponseError,
15420216e5SHanoh Haim    InvalidHeader,
16420216e5SHanoh Haim)
17420216e5SHanoh Haimfrom ..packages import six
18420216e5SHanoh Haim
19420216e5SHanoh Haim
20420216e5SHanoh Haimlog = logging.getLogger(__name__)
21420216e5SHanoh Haim
22420216e5SHanoh Haim# Data structure for representing the metadata of requests that result in a retry.
23420216e5SHanoh HaimRequestHistory = namedtuple('RequestHistory', ["method", "url", "error",
24420216e5SHanoh Haim                                               "status", "redirect_location"])
25420216e5SHanoh Haim
26420216e5SHanoh Haim
27420216e5SHanoh Haimclass Retry(object):
28420216e5SHanoh Haim    """ Retry configuration.
29420216e5SHanoh Haim
30420216e5SHanoh Haim    Each retry attempt will create a new Retry object with updated values, so
31420216e5SHanoh Haim    they can be safely reused.
32420216e5SHanoh Haim
33420216e5SHanoh Haim    Retries can be defined as a default for a pool::
34420216e5SHanoh Haim
35420216e5SHanoh Haim        retries = Retry(connect=5, read=2, redirect=5)
36420216e5SHanoh Haim        http = PoolManager(retries=retries)
37420216e5SHanoh Haim        response = http.request('GET', 'http://example.com/')
38420216e5SHanoh Haim
39420216e5SHanoh Haim    Or per-request (which overrides the default for the pool)::
40420216e5SHanoh Haim
41420216e5SHanoh Haim        response = http.request('GET', 'http://example.com/', retries=Retry(10))
42420216e5SHanoh Haim
43420216e5SHanoh Haim    Retries can be disabled by passing ``False``::
44420216e5SHanoh Haim
45420216e5SHanoh Haim        response = http.request('GET', 'http://example.com/', retries=False)
46420216e5SHanoh Haim
47420216e5SHanoh Haim    Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
48420216e5SHanoh Haim    retries are disabled, in which case the causing exception will be raised.
49420216e5SHanoh Haim
50420216e5SHanoh Haim    :param int total:
51420216e5SHanoh Haim        Total number of retries to allow. Takes precedence over other counts.
52420216e5SHanoh Haim
53420216e5SHanoh Haim        Set to ``None`` to remove this constraint and fall back on other
54420216e5SHanoh Haim        counts. It's a good idea to set this to some sensibly-high value to
55420216e5SHanoh Haim        account for unexpected edge cases and avoid infinite retry loops.
56420216e5SHanoh Haim
57420216e5SHanoh Haim        Set to ``0`` to fail on the first retry.
58420216e5SHanoh Haim
59420216e5SHanoh Haim        Set to ``False`` to disable and imply ``raise_on_redirect=False``.
60420216e5SHanoh Haim
61420216e5SHanoh Haim    :param int connect:
62420216e5SHanoh Haim        How many connection-related errors to retry on.
63420216e5SHanoh Haim
64420216e5SHanoh Haim        These are errors raised before the request is sent to the remote server,
65420216e5SHanoh Haim        which we assume has not triggered the server to process the request.
66420216e5SHanoh Haim
67420216e5SHanoh Haim        Set to ``0`` to fail on the first retry of this type.
68420216e5SHanoh Haim
69420216e5SHanoh Haim    :param int read:
70420216e5SHanoh Haim        How many times to retry on read errors.
71420216e5SHanoh Haim
72420216e5SHanoh Haim        These errors are raised after the request was sent to the server, so the
73420216e5SHanoh Haim        request may have side-effects.
74420216e5SHanoh Haim
75420216e5SHanoh Haim        Set to ``0`` to fail on the first retry of this type.
76420216e5SHanoh Haim
77420216e5SHanoh Haim    :param int redirect:
78420216e5SHanoh Haim        How many redirects to perform. Limit this to avoid infinite redirect
79420216e5SHanoh Haim        loops.
80420216e5SHanoh Haim
81420216e5SHanoh Haim        A redirect is a HTTP response with a status code 301, 302, 303, 307 or
82420216e5SHanoh Haim        308.
83420216e5SHanoh Haim
84420216e5SHanoh Haim        Set to ``0`` to fail on the first retry of this type.
85420216e5SHanoh Haim
86420216e5SHanoh Haim        Set to ``False`` to disable and imply ``raise_on_redirect=False``.
87420216e5SHanoh Haim
88420216e5SHanoh Haim    :param iterable method_whitelist:
89420216e5SHanoh Haim        Set of uppercased HTTP method verbs that we should retry on.
90420216e5SHanoh Haim
91420216e5SHanoh Haim        By default, we only retry on methods which are considered to be
92420216e5SHanoh Haim        idempotent (multiple requests with the same parameters end with the
93420216e5SHanoh Haim        same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.
94420216e5SHanoh Haim
95420216e5SHanoh Haim        Set to a ``False`` value to retry on any verb.
96420216e5SHanoh Haim
97420216e5SHanoh Haim    :param iterable status_forcelist:
98420216e5SHanoh Haim        A set of integer HTTP status codes that we should force a retry on.
99420216e5SHanoh Haim        A retry is initiated if the request method is in ``method_whitelist``
100420216e5SHanoh Haim        and the response status code is in ``status_forcelist``.
101420216e5SHanoh Haim
102420216e5SHanoh Haim        By default, this is disabled with ``None``.
103420216e5SHanoh Haim
104420216e5SHanoh Haim    :param float backoff_factor:
105420216e5SHanoh Haim        A backoff factor to apply between attempts after the second try
106420216e5SHanoh Haim        (most errors are resolved immediately by a second try without a
107420216e5SHanoh Haim        delay). urllib3 will sleep for::
108420216e5SHanoh Haim
109420216e5SHanoh Haim            {backoff factor} * (2 ^ ({number of total retries} - 1))
110420216e5SHanoh Haim
111420216e5SHanoh Haim        seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
112420216e5SHanoh Haim        for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
113420216e5SHanoh Haim        than :attr:`Retry.BACKOFF_MAX`.
114420216e5SHanoh Haim
115420216e5SHanoh Haim        By default, backoff is disabled (set to 0).
116420216e5SHanoh Haim
117420216e5SHanoh Haim    :param bool raise_on_redirect: Whether, if the number of redirects is
118420216e5SHanoh Haim        exhausted, to raise a MaxRetryError, or to return a response with a
119420216e5SHanoh Haim        response code in the 3xx range.
120420216e5SHanoh Haim
121420216e5SHanoh Haim    :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
122420216e5SHanoh Haim        whether we should raise an exception, or return a response,
123420216e5SHanoh Haim        if status falls in ``status_forcelist`` range and retries have
124420216e5SHanoh Haim        been exhausted.
125420216e5SHanoh Haim
126420216e5SHanoh Haim    :param tuple history: The history of the request encountered during
127420216e5SHanoh Haim        each call to :meth:`~Retry.increment`. The list is in the order
128420216e5SHanoh Haim        the requests occurred. Each list item is of class :class:`RequestHistory`.
129420216e5SHanoh Haim
130420216e5SHanoh Haim    :param bool respect_retry_after_header:
131420216e5SHanoh Haim        Whether to respect Retry-After header on status codes defined as
132420216e5SHanoh Haim        :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
133420216e5SHanoh Haim
134420216e5SHanoh Haim    """
135420216e5SHanoh Haim
136420216e5SHanoh Haim    DEFAULT_METHOD_WHITELIST = frozenset([
137420216e5SHanoh Haim        'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
138420216e5SHanoh Haim
139420216e5SHanoh Haim    RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
140420216e5SHanoh Haim
141420216e5SHanoh Haim    #: Maximum backoff time.
142420216e5SHanoh Haim    BACKOFF_MAX = 120
143420216e5SHanoh Haim
144420216e5SHanoh Haim    def __init__(self, total=10, connect=None, read=None, redirect=None,
145420216e5SHanoh Haim                 method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
146420216e5SHanoh Haim                 backoff_factor=0, raise_on_redirect=True, raise_on_status=True,
147420216e5SHanoh Haim                 history=None, respect_retry_after_header=True):
148420216e5SHanoh Haim
149420216e5SHanoh Haim        self.total = total
150420216e5SHanoh Haim        self.connect = connect
151420216e5SHanoh Haim        self.read = read
152420216e5SHanoh Haim
153420216e5SHanoh Haim        if redirect is False or total is False:
154420216e5SHanoh Haim            redirect = 0
155420216e5SHanoh Haim            raise_on_redirect = False
156420216e5SHanoh Haim
157420216e5SHanoh Haim        self.redirect = redirect
158420216e5SHanoh Haim        self.status_forcelist = status_forcelist or set()
159420216e5SHanoh Haim        self.method_whitelist = method_whitelist
160420216e5SHanoh Haim        self.backoff_factor = backoff_factor
161420216e5SHanoh Haim        self.raise_on_redirect = raise_on_redirect
162420216e5SHanoh Haim        self.raise_on_status = raise_on_status
163420216e5SHanoh Haim        self.history = history or tuple()
164420216e5SHanoh Haim        self.respect_retry_after_header = respect_retry_after_header
165420216e5SHanoh Haim
166420216e5SHanoh Haim    def new(self, **kw):
167420216e5SHanoh Haim        params = dict(
168420216e5SHanoh Haim            total=self.total,
169420216e5SHanoh Haim            connect=self.connect, read=self.read, redirect=self.redirect,
170420216e5SHanoh Haim            method_whitelist=self.method_whitelist,
171420216e5SHanoh Haim            status_forcelist=self.status_forcelist,
172420216e5SHanoh Haim            backoff_factor=self.backoff_factor,
173420216e5SHanoh Haim            raise_on_redirect=self.raise_on_redirect,
174420216e5SHanoh Haim            raise_on_status=self.raise_on_status,
175420216e5SHanoh Haim            history=self.history,
176420216e5SHanoh Haim        )
177420216e5SHanoh Haim        params.update(kw)
178420216e5SHanoh Haim        return type(self)(**params)
179420216e5SHanoh Haim
180420216e5SHanoh Haim    @classmethod
181420216e5SHanoh Haim    def from_int(cls, retries, redirect=True, default=None):
182420216e5SHanoh Haim        """ Backwards-compatibility for the old retries format."""
183420216e5SHanoh Haim        if retries is None:
184420216e5SHanoh Haim            retries = default if default is not None else cls.DEFAULT
185420216e5SHanoh Haim
186420216e5SHanoh Haim        if isinstance(retries, Retry):
187420216e5SHanoh Haim            return retries
188420216e5SHanoh Haim
189420216e5SHanoh Haim        redirect = bool(redirect) and None
190420216e5SHanoh Haim        new_retries = cls(retries, redirect=redirect)
191420216e5SHanoh Haim        log.debug("Converted retries value: %r -> %r", retries, new_retries)
192420216e5SHanoh Haim        return new_retries
193420216e5SHanoh Haim
194420216e5SHanoh Haim    def get_backoff_time(self):
195420216e5SHanoh Haim        """ Formula for computing the current backoff
196420216e5SHanoh Haim
197420216e5SHanoh Haim        :rtype: float
198420216e5SHanoh Haim        """
199420216e5SHanoh Haim        # We want to consider only the last consecutive errors sequence (Ignore redirects).
200420216e5SHanoh Haim        consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None,
201420216e5SHanoh Haim                                                    reversed(self.history))))
202420216e5SHanoh Haim        if consecutive_errors_len <= 1:
203420216e5SHanoh Haim            return 0
204420216e5SHanoh Haim
205420216e5SHanoh Haim        backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
206420216e5SHanoh Haim        return min(self.BACKOFF_MAX, backoff_value)
207420216e5SHanoh Haim
208420216e5SHanoh Haim    def parse_retry_after(self, retry_after):
209420216e5SHanoh Haim        # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
210420216e5SHanoh Haim        if re.match(r"^\s*[0-9]+\s*$", retry_after):
211420216e5SHanoh Haim            seconds = int(retry_after)
212420216e5SHanoh Haim        else:
213420216e5SHanoh Haim            retry_date_tuple = email.utils.parsedate(retry_after)
214420216e5SHanoh Haim            if retry_date_tuple is None:
215420216e5SHanoh Haim                raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
216420216e5SHanoh Haim            retry_date = time.mktime(retry_date_tuple)
217420216e5SHanoh Haim            seconds = retry_date - time.time()
218420216e5SHanoh Haim
219420216e5SHanoh Haim        if seconds < 0:
220420216e5SHanoh Haim            seconds = 0
221420216e5SHanoh Haim
222420216e5SHanoh Haim        return seconds
223420216e5SHanoh Haim
224420216e5SHanoh Haim    def get_retry_after(self, response):
225420216e5SHanoh Haim        """ Get the value of Retry-After in seconds. """
226420216e5SHanoh Haim
227420216e5SHanoh Haim        retry_after = response.getheader("Retry-After")
228420216e5SHanoh Haim
229420216e5SHanoh Haim        if retry_after is None:
230420216e5SHanoh Haim            return None
231420216e5SHanoh Haim
232420216e5SHanoh Haim        return self.parse_retry_after(retry_after)
233420216e5SHanoh Haim
234420216e5SHanoh Haim    def sleep_for_retry(self, response=None):
235420216e5SHanoh Haim        retry_after = self.get_retry_after(response)
236420216e5SHanoh Haim        if retry_after:
237420216e5SHanoh Haim            time.sleep(retry_after)
238420216e5SHanoh Haim            return True
239420216e5SHanoh Haim
240420216e5SHanoh Haim        return False
241420216e5SHanoh Haim
242420216e5SHanoh Haim    def _sleep_backoff(self):
243420216e5SHanoh Haim        backoff = self.get_backoff_time()
244420216e5SHanoh Haim        if backoff <= 0:
245420216e5SHanoh Haim            return
246420216e5SHanoh Haim        time.sleep(backoff)
247420216e5SHanoh Haim
248420216e5SHanoh Haim    def sleep(self, response=None):
249420216e5SHanoh Haim        """ Sleep between retry attempts.
250420216e5SHanoh Haim
251420216e5SHanoh Haim        This method will respect a server's ``Retry-After`` response header
252420216e5SHanoh Haim        and sleep the duration of the time requested. If that is not present, it
253420216e5SHanoh Haim        will use an exponential backoff. By default, the backoff factor is 0 and
254420216e5SHanoh Haim        this method will return immediately.
255420216e5SHanoh Haim        """
256420216e5SHanoh Haim
257420216e5SHanoh Haim        if response:
258420216e5SHanoh Haim            slept = self.sleep_for_retry(response)
259420216e5SHanoh Haim            if slept:
260420216e5SHanoh Haim                return
261420216e5SHanoh Haim
262420216e5SHanoh Haim        self._sleep_backoff()
263420216e5SHanoh Haim
264420216e5SHanoh Haim    def _is_connection_error(self, err):
265420216e5SHanoh Haim        """ Errors when we're fairly sure that the server did not receive the
266420216e5SHanoh Haim        request, so it should be safe to retry.
267420216e5SHanoh Haim        """
268420216e5SHanoh Haim        return isinstance(err, ConnectTimeoutError)
269420216e5SHanoh Haim
270420216e5SHanoh Haim    def _is_read_error(self, err):
271420216e5SHanoh Haim        """ Errors that occur after the request has been started, so we should
272420216e5SHanoh Haim        assume that the server began processing it.
273420216e5SHanoh Haim        """
274420216e5SHanoh Haim        return isinstance(err, (ReadTimeoutError, ProtocolError))
275420216e5SHanoh Haim
276420216e5SHanoh Haim    def is_retry(self, method, status_code, has_retry_after=False):
277420216e5SHanoh Haim        """ Is this method/status code retryable? (Based on method/codes whitelists)
278420216e5SHanoh Haim        """
279420216e5SHanoh Haim        if self.method_whitelist and method.upper() not in self.method_whitelist:
280420216e5SHanoh Haim            return False
281420216e5SHanoh Haim
282420216e5SHanoh Haim        if self.status_forcelist and status_code in self.status_forcelist:
283420216e5SHanoh Haim            return True
284420216e5SHanoh Haim
285420216e5SHanoh Haim        return (self.total and self.respect_retry_after_header and
286420216e5SHanoh Haim                has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES))
287420216e5SHanoh Haim
288420216e5SHanoh Haim    def is_exhausted(self):
289420216e5SHanoh Haim        """ Are we out of retries? """
290420216e5SHanoh Haim        retry_counts = (self.total, self.connect, self.read, self.redirect)
291420216e5SHanoh Haim        retry_counts = list(filter(None, retry_counts))
292420216e5SHanoh Haim        if not retry_counts:
293420216e5SHanoh Haim            return False
294420216e5SHanoh Haim
295420216e5SHanoh Haim        return min(retry_counts) < 0
296420216e5SHanoh Haim
297420216e5SHanoh Haim    def increment(self, method=None, url=None, response=None, error=None,
298420216e5SHanoh Haim                  _pool=None, _stacktrace=None):
299420216e5SHanoh Haim        """ Return a new Retry object with incremented retry counters.
300420216e5SHanoh Haim
301420216e5SHanoh Haim        :param response: A response object, or None, if the server did not
302420216e5SHanoh Haim            return a response.
303420216e5SHanoh Haim        :type response: :class:`~urllib3.response.HTTPResponse`
304420216e5SHanoh Haim        :param Exception error: An error encountered during the request, or
305420216e5SHanoh Haim            None if the response was received successfully.
306420216e5SHanoh Haim
307420216e5SHanoh Haim        :return: A new ``Retry`` object.
308420216e5SHanoh Haim        """
309420216e5SHanoh Haim        if self.total is False and error:
310420216e5SHanoh Haim            # Disabled, indicate to re-raise the error.
311420216e5SHanoh Haim            raise six.reraise(type(error), error, _stacktrace)
312420216e5SHanoh Haim
313420216e5SHanoh Haim        total = self.total
314420216e5SHanoh Haim        if total is not None:
315420216e5SHanoh Haim            total -= 1
316420216e5SHanoh Haim
317420216e5SHanoh Haim        connect = self.connect
318420216e5SHanoh Haim        read = self.read
319420216e5SHanoh Haim        redirect = self.redirect
320420216e5SHanoh Haim        cause = 'unknown'
321420216e5SHanoh Haim        status = None
322420216e5SHanoh Haim        redirect_location = None
323420216e5SHanoh Haim
324420216e5SHanoh Haim        if error and self._is_connection_error(error):
325420216e5SHanoh Haim            # Connect retry?
326420216e5SHanoh Haim            if connect is False:
327420216e5SHanoh Haim                raise six.reraise(type(error), error, _stacktrace)
328420216e5SHanoh Haim            elif connect is not None:
329420216e5SHanoh Haim                connect -= 1
330420216e5SHanoh Haim
331420216e5SHanoh Haim        elif error and self._is_read_error(error):
332420216e5SHanoh Haim            # Read retry?
333420216e5SHanoh Haim            if read is False:
334420216e5SHanoh Haim                raise six.reraise(type(error), error, _stacktrace)
335420216e5SHanoh Haim            elif read is not None:
336420216e5SHanoh Haim                read -= 1
337420216e5SHanoh Haim
338420216e5SHanoh Haim        elif response and response.get_redirect_location():
339420216e5SHanoh Haim            # Redirect retry?
340420216e5SHanoh Haim            if redirect is not None:
341420216e5SHanoh Haim                redirect -= 1
342420216e5SHanoh Haim            cause = 'too many redirects'
343420216e5SHanoh Haim            redirect_location = response.get_redirect_location()
344420216e5SHanoh Haim            status = response.status
345420216e5SHanoh Haim
346420216e5SHanoh Haim        else:
347420216e5SHanoh Haim            # Incrementing because of a server error like a 500 in
348420216e5SHanoh Haim            # status_forcelist and a the given method is in the whitelist
349420216e5SHanoh Haim            cause = ResponseError.GENERIC_ERROR
350420216e5SHanoh Haim            if response and response.status:
351420216e5SHanoh Haim                cause = ResponseError.SPECIFIC_ERROR.format(
352420216e5SHanoh Haim                    status_code=response.status)
353420216e5SHanoh Haim                status = response.status
354420216e5SHanoh Haim
355420216e5SHanoh Haim        history = self.history + (RequestHistory(method, url, error, status, redirect_location),)
356420216e5SHanoh Haim
357420216e5SHanoh Haim        new_retry = self.new(
358420216e5SHanoh Haim            total=total,
359420216e5SHanoh Haim            connect=connect, read=read, redirect=redirect,
360420216e5SHanoh Haim            history=history)
361420216e5SHanoh Haim
362420216e5SHanoh Haim        if new_retry.is_exhausted():
363420216e5SHanoh Haim            raise MaxRetryError(_pool, url, error or ResponseError(cause))
364420216e5SHanoh Haim
365420216e5SHanoh Haim        log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
366420216e5SHanoh Haim
367420216e5SHanoh Haim        return new_retry
368420216e5SHanoh Haim
369420216e5SHanoh Haim    def __repr__(self):
370420216e5SHanoh Haim        return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
371420216e5SHanoh Haim                'read={self.read}, redirect={self.redirect})').format(
372420216e5SHanoh Haim                    cls=type(self), self=self)
373420216e5SHanoh Haim
374420216e5SHanoh Haim
375420216e5SHanoh Haim# For backwards compatibility (equivalent to pre-v1.9):
376420216e5SHanoh HaimRetry.DEFAULT = Retry(3)
377