xref: /unit/test/unit/http.py (revision 1635)
11355St.nateldemoura@f5.comimport binascii
21355St.nateldemoura@f5.comimport io
31477Szelenkov@nginx.comimport json
41355St.nateldemoura@f5.comimport os
51019Szelenkov@nginx.comimport re
61477Szelenkov@nginx.comimport select
71477Szelenkov@nginx.comimport socket
81272Szelenkov@nginx.comimport time
91477Szelenkov@nginx.com
10*1635Szelenkov@nginx.comimport pytest
11*1635Szelenkov@nginx.comfrom conftest import option
121019Szelenkov@nginx.comfrom unit.main import TestUnit
131019Szelenkov@nginx.com
141019Szelenkov@nginx.com
151019Szelenkov@nginx.comclass TestHTTP(TestUnit):
161019Szelenkov@nginx.com    def http(self, start_str, **kwargs):
171608Smax.romanov@nginx.com        sock_type = kwargs.get('sock_type', 'ipv4')
181608Smax.romanov@nginx.com        port = kwargs.get('port', 7080)
191608Smax.romanov@nginx.com        url = kwargs.get('url', '/')
201019Szelenkov@nginx.com        http = 'HTTP/1.0' if 'http_10' in kwargs else 'HTTP/1.1'
211019Szelenkov@nginx.com
221608Smax.romanov@nginx.com        headers = kwargs.get('headers',
231608Smax.romanov@nginx.com                             {'Host': 'localhost', 'Connection': 'close'})
241019Szelenkov@nginx.com
251608Smax.romanov@nginx.com        body = kwargs.get('body', b'')
261019Szelenkov@nginx.com        crlf = '\r\n'
271019Szelenkov@nginx.com
281019Szelenkov@nginx.com        if 'addr' not in kwargs:
291019Szelenkov@nginx.com            addr = '::1' if sock_type == 'ipv6' else '127.0.0.1'
301019Szelenkov@nginx.com        else:
311019Szelenkov@nginx.com            addr = kwargs['addr']
321019Szelenkov@nginx.com
331019Szelenkov@nginx.com        sock_types = {
341019Szelenkov@nginx.com            'ipv4': socket.AF_INET,
351019Szelenkov@nginx.com            'ipv6': socket.AF_INET6,
361019Szelenkov@nginx.com            'unix': socket.AF_UNIX,
371019Szelenkov@nginx.com        }
381019Szelenkov@nginx.com
391019Szelenkov@nginx.com        if 'sock' not in kwargs:
401019Szelenkov@nginx.com            sock = socket.socket(sock_types[sock_type], socket.SOCK_STREAM)
411019Szelenkov@nginx.com
421019Szelenkov@nginx.com            if (
431019Szelenkov@nginx.com                sock_type == sock_types['ipv4']
441019Szelenkov@nginx.com                or sock_type == sock_types['ipv6']
451019Szelenkov@nginx.com            ):
461019Szelenkov@nginx.com                sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
471019Szelenkov@nginx.com
481019Szelenkov@nginx.com            if 'wrapper' in kwargs:
491019Szelenkov@nginx.com                sock = kwargs['wrapper'](sock)
501019Szelenkov@nginx.com
511019Szelenkov@nginx.com            connect_args = addr if sock_type == 'unix' else (addr, port)
521019Szelenkov@nginx.com            try:
531019Szelenkov@nginx.com                sock.connect(connect_args)
541019Szelenkov@nginx.com            except ConnectionRefusedError:
551019Szelenkov@nginx.com                sock.close()
561596Szelenkov@nginx.com                pytest.fail('Client can\'t connect to the server.')
571019Szelenkov@nginx.com
581019Szelenkov@nginx.com        else:
591019Szelenkov@nginx.com            sock = kwargs['sock']
601019Szelenkov@nginx.com
611019Szelenkov@nginx.com        if 'raw' not in kwargs:
621019Szelenkov@nginx.com            req = ' '.join([start_str, url, http]) + crlf
631019Szelenkov@nginx.com
641256Szelenkov@nginx.com            if body != b'':
651019Szelenkov@nginx.com                if isinstance(body, str):
661019Szelenkov@nginx.com                    body = body.encode()
671355St.nateldemoura@f5.com                elif isinstance(body, dict):
681355St.nateldemoura@f5.com                    body, content_type = self.form_encode(body)
691355St.nateldemoura@f5.com
701355St.nateldemoura@f5.com                    headers['Content-Type'] = content_type
711019Szelenkov@nginx.com
721019Szelenkov@nginx.com                if 'Content-Length' not in headers:
731019Szelenkov@nginx.com                    headers['Content-Length'] = len(body)
741019Szelenkov@nginx.com
751019Szelenkov@nginx.com            for header, value in headers.items():
761019Szelenkov@nginx.com                if isinstance(value, list):
771019Szelenkov@nginx.com                    for v in value:
781019Szelenkov@nginx.com                        req += header + ': ' + str(v) + crlf
791019Szelenkov@nginx.com
801019Szelenkov@nginx.com                else:
811019Szelenkov@nginx.com                    req += header + ': ' + str(value) + crlf
821019Szelenkov@nginx.com
831019Szelenkov@nginx.com            req = (req + crlf).encode() + body
841019Szelenkov@nginx.com
851019Szelenkov@nginx.com        else:
861019Szelenkov@nginx.com            req = start_str
871019Szelenkov@nginx.com
881019Szelenkov@nginx.com        sock.sendall(req)
891019Szelenkov@nginx.com
901608Smax.romanov@nginx.com        encoding = kwargs.get('encoding', 'utf-8')
911295St.nateldemoura@f5.com
921372Szelenkov@nginx.com        self.log_out(req, encoding)
931019Szelenkov@nginx.com
941019Szelenkov@nginx.com        resp = ''
951019Szelenkov@nginx.com
961019Szelenkov@nginx.com        if 'no_recv' not in kwargs:
971424Szelenkov@nginx.com            recvall_kwargs = {}
981424Szelenkov@nginx.com
991424Szelenkov@nginx.com            if 'read_timeout' in kwargs:
1001424Szelenkov@nginx.com                recvall_kwargs['read_timeout'] = kwargs['read_timeout']
1011424Szelenkov@nginx.com
1021424Szelenkov@nginx.com            if 'read_buffer_size' in kwargs:
1031424Szelenkov@nginx.com                recvall_kwargs['buff_size'] = kwargs['read_buffer_size']
1041424Szelenkov@nginx.com
1051424Szelenkov@nginx.com            resp = self.recvall(sock, **recvall_kwargs).decode(encoding)
1061019Szelenkov@nginx.com
1071372Szelenkov@nginx.com        self.log_in(resp)
1081019Szelenkov@nginx.com
1091019Szelenkov@nginx.com        if 'raw_resp' not in kwargs:
1101019Szelenkov@nginx.com            resp = self._resp_to_dict(resp)
1111019Szelenkov@nginx.com
1121295St.nateldemoura@f5.com            headers = resp.get('headers')
1131295St.nateldemoura@f5.com            if headers and headers.get('Transfer-Encoding') == 'chunked':
1141295St.nateldemoura@f5.com                resp['body'] = self._parse_chunked_body(resp['body']).decode(
1151295St.nateldemoura@f5.com                    encoding
1161295St.nateldemoura@f5.com                )
1171295St.nateldemoura@f5.com
1181296St.nateldemoura@f5.com            if 'json' in kwargs:
1191296St.nateldemoura@f5.com                resp = self._parse_json(resp)
1201296St.nateldemoura@f5.com
1211019Szelenkov@nginx.com        if 'start' not in kwargs:
1221019Szelenkov@nginx.com            sock.close()
1231019Szelenkov@nginx.com            return resp
1241019Szelenkov@nginx.com
1251019Szelenkov@nginx.com        return (resp, sock)
1261019Szelenkov@nginx.com
1271372Szelenkov@nginx.com    def log_out(self, log, encoding):
1281596Szelenkov@nginx.com        if option.detailed:
1291372Szelenkov@nginx.com            print('>>>')
1301372Szelenkov@nginx.com            log = self.log_truncate(log)
1311372Szelenkov@nginx.com            try:
1321372Szelenkov@nginx.com                print(log.decode(encoding, 'ignore'))
1331372Szelenkov@nginx.com            except UnicodeEncodeError:
1341372Szelenkov@nginx.com                print(log)
1351372Szelenkov@nginx.com
1361372Szelenkov@nginx.com    def log_in(self, log):
1371596Szelenkov@nginx.com        if option.detailed:
1381372Szelenkov@nginx.com            print('<<<')
1391372Szelenkov@nginx.com            log = self.log_truncate(log)
1401372Szelenkov@nginx.com            try:
1411372Szelenkov@nginx.com                print(log)
1421372Szelenkov@nginx.com            except UnicodeEncodeError:
1431372Szelenkov@nginx.com                print(log.encode())
1441372Szelenkov@nginx.com
1451372Szelenkov@nginx.com    def log_truncate(self, log, limit=1024):
1461372Szelenkov@nginx.com        len_log = len(log)
1471372Szelenkov@nginx.com        if len_log > limit:
1481372Szelenkov@nginx.com            log = log[:limit]
1491372Szelenkov@nginx.com            appendix = '(...logged %s of %s bytes)' % (limit, len_log)
1501372Szelenkov@nginx.com
1511372Szelenkov@nginx.com            if isinstance(log, bytes):
1521372Szelenkov@nginx.com                appendix = appendix.encode()
1531372Szelenkov@nginx.com
1541372Szelenkov@nginx.com            log = log + appendix
1551372Szelenkov@nginx.com
1561372Szelenkov@nginx.com        return log
1571372Szelenkov@nginx.com
1581019Szelenkov@nginx.com    def delete(self, **kwargs):
1591019Szelenkov@nginx.com        return self.http('DELETE', **kwargs)
1601019Szelenkov@nginx.com
1611019Szelenkov@nginx.com    def get(self, **kwargs):
1621019Szelenkov@nginx.com        return self.http('GET', **kwargs)
1631019Szelenkov@nginx.com
1641172Szelenkov@nginx.com    def head(self, **kwargs):
1651172Szelenkov@nginx.com        return self.http('HEAD', **kwargs)
1661172Szelenkov@nginx.com
1671019Szelenkov@nginx.com    def post(self, **kwargs):
1681019Szelenkov@nginx.com        return self.http('POST', **kwargs)
1691019Szelenkov@nginx.com
1701019Szelenkov@nginx.com    def put(self, **kwargs):
1711019Szelenkov@nginx.com        return self.http('PUT', **kwargs)
1721019Szelenkov@nginx.com
1731424Szelenkov@nginx.com    def recvall(self, sock, **kwargs):
1741444Szelenkov@nginx.com        timeout_default = 60
1751444Szelenkov@nginx.com
1761608Smax.romanov@nginx.com        timeout = kwargs.get('read_timeout', timeout_default)
1771608Smax.romanov@nginx.com        buff_size = kwargs.get('buff_size', 4096)
1781424Szelenkov@nginx.com
1791019Szelenkov@nginx.com        data = b''
1801444Szelenkov@nginx.com        while True:
1811444Szelenkov@nginx.com            rlist = select.select([sock], [], [], timeout)[0]
1821444Szelenkov@nginx.com            if not rlist:
1831444Szelenkov@nginx.com                # For all current cases if the "read_timeout" was changed
1841444Szelenkov@nginx.com                # than test do not expect to get a response from server.
1851444Szelenkov@nginx.com                if timeout == timeout_default:
1861596Szelenkov@nginx.com                    pytest.fail('Can\'t read response from server.')
1871444Szelenkov@nginx.com                break
1881444Szelenkov@nginx.com
1891019Szelenkov@nginx.com            try:
1901019Szelenkov@nginx.com                part = sock.recv(buff_size)
1911019Szelenkov@nginx.com            except:
1921019Szelenkov@nginx.com                break
1931019Szelenkov@nginx.com
1941019Szelenkov@nginx.com            data += part
1951019Szelenkov@nginx.com
1961019Szelenkov@nginx.com            if not len(part):
1971019Szelenkov@nginx.com                break
1981019Szelenkov@nginx.com
1991019Szelenkov@nginx.com        return data
2001019Szelenkov@nginx.com
2011019Szelenkov@nginx.com    def _resp_to_dict(self, resp):
2021295St.nateldemoura@f5.com        m = re.search(r'(.*?\x0d\x0a?)\x0d\x0a?(.*)', resp, re.M | re.S)
2031019Szelenkov@nginx.com
2041019Szelenkov@nginx.com        if not m:
2051019Szelenkov@nginx.com            return {}
2061019Szelenkov@nginx.com
2071019Szelenkov@nginx.com        headers_text, body = m.group(1), m.group(2)
2081019Szelenkov@nginx.com
2091019Szelenkov@nginx.com        p = re.compile('(.*?)\x0d\x0a?', re.M | re.S)
2101019Szelenkov@nginx.com        headers_lines = p.findall(headers_text)
2111019Szelenkov@nginx.com
2121019Szelenkov@nginx.com        status = re.search(
2131295St.nateldemoura@f5.com            r'^HTTP\/\d\.\d\s(\d+)|$', headers_lines.pop(0)
2141019Szelenkov@nginx.com        ).group(1)
2151019Szelenkov@nginx.com
2161019Szelenkov@nginx.com        headers = {}
2171019Szelenkov@nginx.com        for line in headers_lines:
2181295St.nateldemoura@f5.com            m = re.search(r'(.*)\:\s(.*)', line)
2191019Szelenkov@nginx.com
2201019Szelenkov@nginx.com            if m.group(1) not in headers:
2211019Szelenkov@nginx.com                headers[m.group(1)] = m.group(2)
2221019Szelenkov@nginx.com
2231019Szelenkov@nginx.com            elif isinstance(headers[m.group(1)], list):
2241019Szelenkov@nginx.com                headers[m.group(1)].append(m.group(2))
2251019Szelenkov@nginx.com
2261019Szelenkov@nginx.com            else:
2271019Szelenkov@nginx.com                headers[m.group(1)] = [headers[m.group(1)], m.group(2)]
2281019Szelenkov@nginx.com
2291019Szelenkov@nginx.com        return {'status': int(status), 'headers': headers, 'body': body}
2301272Szelenkov@nginx.com
2311295St.nateldemoura@f5.com    def _parse_chunked_body(self, raw_body):
2321295St.nateldemoura@f5.com        if isinstance(raw_body, str):
2331295St.nateldemoura@f5.com            raw_body = bytes(raw_body.encode())
2341295St.nateldemoura@f5.com
2351295St.nateldemoura@f5.com        crlf = b'\r\n'
2361295St.nateldemoura@f5.com        chunks = raw_body.split(crlf)
2371295St.nateldemoura@f5.com
2381295St.nateldemoura@f5.com        if len(chunks) < 3:
2391596Szelenkov@nginx.com            pytest.fail('Invalid chunked body')
2401295St.nateldemoura@f5.com
2411295St.nateldemoura@f5.com        if chunks.pop() != b'':
2421596Szelenkov@nginx.com            pytest.fail('No CRLF at the end of the body')
2431295St.nateldemoura@f5.com
2441295St.nateldemoura@f5.com        try:
2451295St.nateldemoura@f5.com            last_size = int(chunks[-2], 16)
2461295St.nateldemoura@f5.com        except:
2471596Szelenkov@nginx.com            pytest.fail('Invalid zero size chunk')
2481295St.nateldemoura@f5.com
2491295St.nateldemoura@f5.com        if last_size != 0 or chunks[-1] != b'':
2501596Szelenkov@nginx.com            pytest.fail('Incomplete body')
2511295St.nateldemoura@f5.com
2521295St.nateldemoura@f5.com        body = b''
2531295St.nateldemoura@f5.com        while len(chunks) >= 2:
2541295St.nateldemoura@f5.com            try:
2551295St.nateldemoura@f5.com                size = int(chunks.pop(0), 16)
2561295St.nateldemoura@f5.com            except:
2571596Szelenkov@nginx.com                pytest.fail('Invalid chunk size %s' % str(size))
2581295St.nateldemoura@f5.com
2591295St.nateldemoura@f5.com            if size == 0:
2601596Szelenkov@nginx.com                assert len(chunks) == 1, 'last zero size'
2611295St.nateldemoura@f5.com                break
2621295St.nateldemoura@f5.com
2631295St.nateldemoura@f5.com            temp_body = crlf.join(chunks)
2641295St.nateldemoura@f5.com
2651295St.nateldemoura@f5.com            body += temp_body[:size]
2661295St.nateldemoura@f5.com
2671295St.nateldemoura@f5.com            temp_body = temp_body[size + len(crlf) :]
2681295St.nateldemoura@f5.com
2691295St.nateldemoura@f5.com            chunks = temp_body.split(crlf)
2701295St.nateldemoura@f5.com
2711295St.nateldemoura@f5.com        return body
2721295St.nateldemoura@f5.com
2731296St.nateldemoura@f5.com    def _parse_json(self, resp):
2741296St.nateldemoura@f5.com        headers = resp['headers']
2751296St.nateldemoura@f5.com
2761596Szelenkov@nginx.com        assert 'Content-Type' in headers
2771596Szelenkov@nginx.com        assert headers['Content-Type'] == 'application/json'
2781296St.nateldemoura@f5.com
2791296St.nateldemoura@f5.com        resp['body'] = json.loads(resp['body'])
2801296St.nateldemoura@f5.com
2811296St.nateldemoura@f5.com        return resp
2821296St.nateldemoura@f5.com
2831296St.nateldemoura@f5.com    def getjson(self, **kwargs):
2841296St.nateldemoura@f5.com        return self.get(json=True, **kwargs)
2851296St.nateldemoura@f5.com
2861272Szelenkov@nginx.com    def waitforsocket(self, port):
2871272Szelenkov@nginx.com        ret = False
2881272Szelenkov@nginx.com
2891272Szelenkov@nginx.com        for i in range(50):
2901272Szelenkov@nginx.com            try:
2911272Szelenkov@nginx.com                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
2921272Szelenkov@nginx.com                sock.connect(('127.0.0.1', port))
2931272Szelenkov@nginx.com                ret = True
2941272Szelenkov@nginx.com                break
2951272Szelenkov@nginx.com            except:
2961272Szelenkov@nginx.com                sock.close()
2971272Szelenkov@nginx.com                time.sleep(0.1)
2981272Szelenkov@nginx.com
2991272Szelenkov@nginx.com        sock.close()
3001272Szelenkov@nginx.com
3011596Szelenkov@nginx.com        assert ret, 'socket connected'
3021355St.nateldemoura@f5.com
3031355St.nateldemoura@f5.com    def form_encode(self, fields):
3041355St.nateldemoura@f5.com        is_multipart = False
3051355St.nateldemoura@f5.com
3061355St.nateldemoura@f5.com        for _, value in fields.items():
3071355St.nateldemoura@f5.com            if isinstance(value, dict):
3081355St.nateldemoura@f5.com                is_multipart = True
3091355St.nateldemoura@f5.com                break
3101355St.nateldemoura@f5.com
3111355St.nateldemoura@f5.com        if is_multipart:
3121355St.nateldemoura@f5.com            body, content_type = self.multipart_encode(fields)
3131355St.nateldemoura@f5.com
3141355St.nateldemoura@f5.com        else:
3151355St.nateldemoura@f5.com            body, content_type = self.form_url_encode(fields)
3161355St.nateldemoura@f5.com
3171355St.nateldemoura@f5.com        return body, content_type
3181355St.nateldemoura@f5.com
3191355St.nateldemoura@f5.com    def form_url_encode(self, fields):
3201355St.nateldemoura@f5.com        data = "&".join("%s=%s" % (name, value)
3211355St.nateldemoura@f5.com                        for name, value in fields.items()).encode()
3221355St.nateldemoura@f5.com        return data, 'application/x-www-form-urlencoded'
3231355St.nateldemoura@f5.com
3241355St.nateldemoura@f5.com    def multipart_encode(self, fields):
3251355St.nateldemoura@f5.com        boundary = binascii.hexlify(os.urandom(16)).decode('ascii')
3261355St.nateldemoura@f5.com
3271355St.nateldemoura@f5.com        body = ''
3281355St.nateldemoura@f5.com
3291355St.nateldemoura@f5.com        for field, value in fields.items():
3301355St.nateldemoura@f5.com            filename = ''
3311355St.nateldemoura@f5.com            datatype = ''
3321355St.nateldemoura@f5.com
3331355St.nateldemoura@f5.com            if isinstance(value, dict):
3341355St.nateldemoura@f5.com                datatype = 'text/plain'
3351355St.nateldemoura@f5.com                filename = value['filename']
3361355St.nateldemoura@f5.com
3371355St.nateldemoura@f5.com                if value.get('type'):
3381355St.nateldemoura@f5.com                    datatype = value['type']
3391355St.nateldemoura@f5.com
3401355St.nateldemoura@f5.com                if not isinstance(value['data'], io.IOBase):
3411596Szelenkov@nginx.com                    pytest.fail('multipart encoding of file requires a stream.')
3421355St.nateldemoura@f5.com
3431355St.nateldemoura@f5.com                data = value['data'].read()
3441355St.nateldemoura@f5.com
3451355St.nateldemoura@f5.com            elif isinstance(value, str):
3461355St.nateldemoura@f5.com                data = value
3471355St.nateldemoura@f5.com
3481355St.nateldemoura@f5.com            else:
3491596Szelenkov@nginx.com                pytest.fail('multipart requires a string or stream data')
3501355St.nateldemoura@f5.com
3511355St.nateldemoura@f5.com            body += (
3521355St.nateldemoura@f5.com                "--%s\r\nContent-Disposition: form-data; name=\"%s\""
3531355St.nateldemoura@f5.com            ) % (boundary, field)
3541355St.nateldemoura@f5.com
3551355St.nateldemoura@f5.com            if filename != '':
3561355St.nateldemoura@f5.com                body += "; filename=\"%s\"" % filename
3571355St.nateldemoura@f5.com
3581355St.nateldemoura@f5.com            body += "\r\n"
3591355St.nateldemoura@f5.com
3601355St.nateldemoura@f5.com            if datatype != '':
3611355St.nateldemoura@f5.com                body += "Content-Type: %s\r\n" % datatype
3621355St.nateldemoura@f5.com
3631355St.nateldemoura@f5.com            body += "\r\n%s\r\n" % data
3641355St.nateldemoura@f5.com
3651355St.nateldemoura@f5.com        body += "--%s--\r\n" % boundary
3661355St.nateldemoura@f5.com
3671355St.nateldemoura@f5.com        return body.encode(), "multipart/form-data; boundary=%s" % boundary
368