11355St.nateldemoura@f5.comimport binascii 21355St.nateldemoura@f5.comimport io 31477Szelenkov@nginx.comimport json 41355St.nateldemoura@f5.comimport os 51019Szelenkov@nginx.comimport re 61477Szelenkov@nginx.comimport select 71477Szelenkov@nginx.comimport socket 81477Szelenkov@nginx.com 91635Szelenkov@nginx.comimport pytest 101019Szelenkov@nginx.comfrom unit.main import TestUnit 11*1730Szelenkov@nginx.comfrom unit.option import option 121019Szelenkov@nginx.com 131019Szelenkov@nginx.com 141019Szelenkov@nginx.comclass TestHTTP(TestUnit): 151019Szelenkov@nginx.com def http(self, start_str, **kwargs): 161608Smax.romanov@nginx.com sock_type = kwargs.get('sock_type', 'ipv4') 171608Smax.romanov@nginx.com port = kwargs.get('port', 7080) 181608Smax.romanov@nginx.com url = kwargs.get('url', '/') 191019Szelenkov@nginx.com http = 'HTTP/1.0' if 'http_10' in kwargs else 'HTTP/1.1' 201019Szelenkov@nginx.com 211608Smax.romanov@nginx.com headers = kwargs.get('headers', 221608Smax.romanov@nginx.com {'Host': 'localhost', 'Connection': 'close'}) 231019Szelenkov@nginx.com 241608Smax.romanov@nginx.com body = kwargs.get('body', b'') 251019Szelenkov@nginx.com crlf = '\r\n' 261019Szelenkov@nginx.com 271019Szelenkov@nginx.com if 'addr' not in kwargs: 281019Szelenkov@nginx.com addr = '::1' if sock_type == 'ipv6' else '127.0.0.1' 291019Szelenkov@nginx.com else: 301019Szelenkov@nginx.com addr = kwargs['addr'] 311019Szelenkov@nginx.com 321019Szelenkov@nginx.com sock_types = { 331019Szelenkov@nginx.com 'ipv4': socket.AF_INET, 341019Szelenkov@nginx.com 'ipv6': socket.AF_INET6, 351019Szelenkov@nginx.com 'unix': socket.AF_UNIX, 361019Szelenkov@nginx.com } 371019Szelenkov@nginx.com 381019Szelenkov@nginx.com if 'sock' not in kwargs: 391019Szelenkov@nginx.com sock = socket.socket(sock_types[sock_type], socket.SOCK_STREAM) 401019Szelenkov@nginx.com 411019Szelenkov@nginx.com if ( 421019Szelenkov@nginx.com sock_type == sock_types['ipv4'] 431019Szelenkov@nginx.com or sock_type == sock_types['ipv6'] 441019Szelenkov@nginx.com ): 451019Szelenkov@nginx.com sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 461019Szelenkov@nginx.com 471019Szelenkov@nginx.com if 'wrapper' in kwargs: 481019Szelenkov@nginx.com sock = kwargs['wrapper'](sock) 491019Szelenkov@nginx.com 501019Szelenkov@nginx.com connect_args = addr if sock_type == 'unix' else (addr, port) 511019Szelenkov@nginx.com try: 521019Szelenkov@nginx.com sock.connect(connect_args) 531019Szelenkov@nginx.com except ConnectionRefusedError: 541019Szelenkov@nginx.com sock.close() 551596Szelenkov@nginx.com pytest.fail('Client can\'t connect to the server.') 561019Szelenkov@nginx.com 571019Szelenkov@nginx.com else: 581019Szelenkov@nginx.com sock = kwargs['sock'] 591019Szelenkov@nginx.com 601019Szelenkov@nginx.com if 'raw' not in kwargs: 611019Szelenkov@nginx.com req = ' '.join([start_str, url, http]) + crlf 621019Szelenkov@nginx.com 631256Szelenkov@nginx.com if body != b'': 641019Szelenkov@nginx.com if isinstance(body, str): 651019Szelenkov@nginx.com body = body.encode() 661355St.nateldemoura@f5.com elif isinstance(body, dict): 671355St.nateldemoura@f5.com body, content_type = self.form_encode(body) 681355St.nateldemoura@f5.com 691355St.nateldemoura@f5.com headers['Content-Type'] = content_type 701019Szelenkov@nginx.com 711019Szelenkov@nginx.com if 'Content-Length' not in headers: 721019Szelenkov@nginx.com headers['Content-Length'] = len(body) 731019Szelenkov@nginx.com 741019Szelenkov@nginx.com for header, value in headers.items(): 751019Szelenkov@nginx.com if isinstance(value, list): 761019Szelenkov@nginx.com for v in value: 771019Szelenkov@nginx.com req += header + ': ' + str(v) + crlf 781019Szelenkov@nginx.com 791019Szelenkov@nginx.com else: 801019Szelenkov@nginx.com req += header + ': ' + str(value) + crlf 811019Szelenkov@nginx.com 821019Szelenkov@nginx.com req = (req + crlf).encode() + body 831019Szelenkov@nginx.com 841019Szelenkov@nginx.com else: 851019Szelenkov@nginx.com req = start_str 861019Szelenkov@nginx.com 871019Szelenkov@nginx.com sock.sendall(req) 881019Szelenkov@nginx.com 891608Smax.romanov@nginx.com encoding = kwargs.get('encoding', 'utf-8') 901295St.nateldemoura@f5.com 911372Szelenkov@nginx.com self.log_out(req, encoding) 921019Szelenkov@nginx.com 931019Szelenkov@nginx.com resp = '' 941019Szelenkov@nginx.com 951019Szelenkov@nginx.com if 'no_recv' not in kwargs: 961424Szelenkov@nginx.com recvall_kwargs = {} 971424Szelenkov@nginx.com 981424Szelenkov@nginx.com if 'read_timeout' in kwargs: 991424Szelenkov@nginx.com recvall_kwargs['read_timeout'] = kwargs['read_timeout'] 1001424Szelenkov@nginx.com 1011424Szelenkov@nginx.com if 'read_buffer_size' in kwargs: 1021424Szelenkov@nginx.com recvall_kwargs['buff_size'] = kwargs['read_buffer_size'] 1031424Szelenkov@nginx.com 1041424Szelenkov@nginx.com resp = self.recvall(sock, **recvall_kwargs).decode(encoding) 1051019Szelenkov@nginx.com 1061372Szelenkov@nginx.com self.log_in(resp) 1071019Szelenkov@nginx.com 1081019Szelenkov@nginx.com if 'raw_resp' not in kwargs: 1091019Szelenkov@nginx.com resp = self._resp_to_dict(resp) 1101019Szelenkov@nginx.com 1111295St.nateldemoura@f5.com headers = resp.get('headers') 1121295St.nateldemoura@f5.com if headers and headers.get('Transfer-Encoding') == 'chunked': 1131295St.nateldemoura@f5.com resp['body'] = self._parse_chunked_body(resp['body']).decode( 1141295St.nateldemoura@f5.com encoding 1151295St.nateldemoura@f5.com ) 1161295St.nateldemoura@f5.com 1171296St.nateldemoura@f5.com if 'json' in kwargs: 1181296St.nateldemoura@f5.com resp = self._parse_json(resp) 1191296St.nateldemoura@f5.com 1201019Szelenkov@nginx.com if 'start' not in kwargs: 1211019Szelenkov@nginx.com sock.close() 1221019Szelenkov@nginx.com return resp 1231019Szelenkov@nginx.com 1241019Szelenkov@nginx.com return (resp, sock) 1251019Szelenkov@nginx.com 1261372Szelenkov@nginx.com def log_out(self, log, encoding): 1271596Szelenkov@nginx.com if option.detailed: 1281372Szelenkov@nginx.com print('>>>') 1291372Szelenkov@nginx.com log = self.log_truncate(log) 1301372Szelenkov@nginx.com try: 1311372Szelenkov@nginx.com print(log.decode(encoding, 'ignore')) 1321372Szelenkov@nginx.com except UnicodeEncodeError: 1331372Szelenkov@nginx.com print(log) 1341372Szelenkov@nginx.com 1351372Szelenkov@nginx.com def log_in(self, log): 1361596Szelenkov@nginx.com if option.detailed: 1371372Szelenkov@nginx.com print('<<<') 1381372Szelenkov@nginx.com log = self.log_truncate(log) 1391372Szelenkov@nginx.com try: 1401372Szelenkov@nginx.com print(log) 1411372Szelenkov@nginx.com except UnicodeEncodeError: 1421372Szelenkov@nginx.com print(log.encode()) 1431372Szelenkov@nginx.com 1441372Szelenkov@nginx.com def log_truncate(self, log, limit=1024): 1451372Szelenkov@nginx.com len_log = len(log) 1461372Szelenkov@nginx.com if len_log > limit: 1471372Szelenkov@nginx.com log = log[:limit] 1481372Szelenkov@nginx.com appendix = '(...logged %s of %s bytes)' % (limit, len_log) 1491372Szelenkov@nginx.com 1501372Szelenkov@nginx.com if isinstance(log, bytes): 1511372Szelenkov@nginx.com appendix = appendix.encode() 1521372Szelenkov@nginx.com 1531372Szelenkov@nginx.com log = log + appendix 1541372Szelenkov@nginx.com 1551372Szelenkov@nginx.com return log 1561372Szelenkov@nginx.com 1571019Szelenkov@nginx.com def delete(self, **kwargs): 1581019Szelenkov@nginx.com return self.http('DELETE', **kwargs) 1591019Szelenkov@nginx.com 1601019Szelenkov@nginx.com def get(self, **kwargs): 1611019Szelenkov@nginx.com return self.http('GET', **kwargs) 1621019Szelenkov@nginx.com 1631172Szelenkov@nginx.com def head(self, **kwargs): 1641172Szelenkov@nginx.com return self.http('HEAD', **kwargs) 1651172Szelenkov@nginx.com 1661019Szelenkov@nginx.com def post(self, **kwargs): 1671019Szelenkov@nginx.com return self.http('POST', **kwargs) 1681019Szelenkov@nginx.com 1691019Szelenkov@nginx.com def put(self, **kwargs): 1701019Szelenkov@nginx.com return self.http('PUT', **kwargs) 1711019Szelenkov@nginx.com 1721424Szelenkov@nginx.com def recvall(self, sock, **kwargs): 1731444Szelenkov@nginx.com timeout_default = 60 1741444Szelenkov@nginx.com 1751608Smax.romanov@nginx.com timeout = kwargs.get('read_timeout', timeout_default) 1761608Smax.romanov@nginx.com buff_size = kwargs.get('buff_size', 4096) 1771424Szelenkov@nginx.com 1781019Szelenkov@nginx.com data = b'' 1791444Szelenkov@nginx.com while True: 1801444Szelenkov@nginx.com rlist = select.select([sock], [], [], timeout)[0] 1811444Szelenkov@nginx.com if not rlist: 1821444Szelenkov@nginx.com # For all current cases if the "read_timeout" was changed 1831444Szelenkov@nginx.com # than test do not expect to get a response from server. 1841444Szelenkov@nginx.com if timeout == timeout_default: 1851596Szelenkov@nginx.com pytest.fail('Can\'t read response from server.') 1861444Szelenkov@nginx.com break 1871444Szelenkov@nginx.com 1881019Szelenkov@nginx.com try: 1891019Szelenkov@nginx.com part = sock.recv(buff_size) 1901706Smax.romanov@nginx.com 1911706Smax.romanov@nginx.com except KeyboardInterrupt: 1921706Smax.romanov@nginx.com raise 1931706Smax.romanov@nginx.com 1941019Szelenkov@nginx.com except: 1951019Szelenkov@nginx.com break 1961019Szelenkov@nginx.com 1971019Szelenkov@nginx.com data += part 1981019Szelenkov@nginx.com 1991019Szelenkov@nginx.com if not len(part): 2001019Szelenkov@nginx.com break 2011019Szelenkov@nginx.com 2021019Szelenkov@nginx.com return data 2031019Szelenkov@nginx.com 2041019Szelenkov@nginx.com def _resp_to_dict(self, resp): 2051295St.nateldemoura@f5.com m = re.search(r'(.*?\x0d\x0a?)\x0d\x0a?(.*)', resp, re.M | re.S) 2061019Szelenkov@nginx.com 2071019Szelenkov@nginx.com if not m: 2081019Szelenkov@nginx.com return {} 2091019Szelenkov@nginx.com 2101019Szelenkov@nginx.com headers_text, body = m.group(1), m.group(2) 2111019Szelenkov@nginx.com 2121019Szelenkov@nginx.com p = re.compile('(.*?)\x0d\x0a?', re.M | re.S) 2131019Szelenkov@nginx.com headers_lines = p.findall(headers_text) 2141019Szelenkov@nginx.com 2151019Szelenkov@nginx.com status = re.search( 2161295St.nateldemoura@f5.com r'^HTTP\/\d\.\d\s(\d+)|$', headers_lines.pop(0) 2171019Szelenkov@nginx.com ).group(1) 2181019Szelenkov@nginx.com 2191019Szelenkov@nginx.com headers = {} 2201019Szelenkov@nginx.com for line in headers_lines: 2211295St.nateldemoura@f5.com m = re.search(r'(.*)\:\s(.*)', line) 2221019Szelenkov@nginx.com 2231019Szelenkov@nginx.com if m.group(1) not in headers: 2241019Szelenkov@nginx.com headers[m.group(1)] = m.group(2) 2251019Szelenkov@nginx.com 2261019Szelenkov@nginx.com elif isinstance(headers[m.group(1)], list): 2271019Szelenkov@nginx.com headers[m.group(1)].append(m.group(2)) 2281019Szelenkov@nginx.com 2291019Szelenkov@nginx.com else: 2301019Szelenkov@nginx.com headers[m.group(1)] = [headers[m.group(1)], m.group(2)] 2311019Szelenkov@nginx.com 2321019Szelenkov@nginx.com return {'status': int(status), 'headers': headers, 'body': body} 2331272Szelenkov@nginx.com 2341295St.nateldemoura@f5.com def _parse_chunked_body(self, raw_body): 2351295St.nateldemoura@f5.com if isinstance(raw_body, str): 2361295St.nateldemoura@f5.com raw_body = bytes(raw_body.encode()) 2371295St.nateldemoura@f5.com 2381295St.nateldemoura@f5.com crlf = b'\r\n' 2391295St.nateldemoura@f5.com chunks = raw_body.split(crlf) 2401295St.nateldemoura@f5.com 2411295St.nateldemoura@f5.com if len(chunks) < 3: 2421596Szelenkov@nginx.com pytest.fail('Invalid chunked body') 2431295St.nateldemoura@f5.com 2441295St.nateldemoura@f5.com if chunks.pop() != b'': 2451596Szelenkov@nginx.com pytest.fail('No CRLF at the end of the body') 2461295St.nateldemoura@f5.com 2471295St.nateldemoura@f5.com try: 2481295St.nateldemoura@f5.com last_size = int(chunks[-2], 16) 2491706Smax.romanov@nginx.com 2501706Smax.romanov@nginx.com except ValueError: 2511596Szelenkov@nginx.com pytest.fail('Invalid zero size chunk') 2521295St.nateldemoura@f5.com 2531295St.nateldemoura@f5.com if last_size != 0 or chunks[-1] != b'': 2541596Szelenkov@nginx.com pytest.fail('Incomplete body') 2551295St.nateldemoura@f5.com 2561295St.nateldemoura@f5.com body = b'' 2571295St.nateldemoura@f5.com while len(chunks) >= 2: 2581295St.nateldemoura@f5.com try: 2591295St.nateldemoura@f5.com size = int(chunks.pop(0), 16) 2601706Smax.romanov@nginx.com 2611706Smax.romanov@nginx.com except ValueError: 2621596Szelenkov@nginx.com pytest.fail('Invalid chunk size %s' % str(size)) 2631295St.nateldemoura@f5.com 2641295St.nateldemoura@f5.com if size == 0: 2651596Szelenkov@nginx.com assert len(chunks) == 1, 'last zero size' 2661295St.nateldemoura@f5.com break 2671295St.nateldemoura@f5.com 2681295St.nateldemoura@f5.com temp_body = crlf.join(chunks) 2691295St.nateldemoura@f5.com 2701295St.nateldemoura@f5.com body += temp_body[:size] 2711295St.nateldemoura@f5.com 2721295St.nateldemoura@f5.com temp_body = temp_body[size + len(crlf) :] 2731295St.nateldemoura@f5.com 2741295St.nateldemoura@f5.com chunks = temp_body.split(crlf) 2751295St.nateldemoura@f5.com 2761295St.nateldemoura@f5.com return body 2771295St.nateldemoura@f5.com 2781296St.nateldemoura@f5.com def _parse_json(self, resp): 2791296St.nateldemoura@f5.com headers = resp['headers'] 2801296St.nateldemoura@f5.com 2811596Szelenkov@nginx.com assert 'Content-Type' in headers 2821596Szelenkov@nginx.com assert headers['Content-Type'] == 'application/json' 2831296St.nateldemoura@f5.com 2841296St.nateldemoura@f5.com resp['body'] = json.loads(resp['body']) 2851296St.nateldemoura@f5.com 2861296St.nateldemoura@f5.com return resp 2871296St.nateldemoura@f5.com 2881296St.nateldemoura@f5.com def getjson(self, **kwargs): 2891296St.nateldemoura@f5.com return self.get(json=True, **kwargs) 2901296St.nateldemoura@f5.com 2911355St.nateldemoura@f5.com def form_encode(self, fields): 2921355St.nateldemoura@f5.com is_multipart = False 2931355St.nateldemoura@f5.com 2941355St.nateldemoura@f5.com for _, value in fields.items(): 2951355St.nateldemoura@f5.com if isinstance(value, dict): 2961355St.nateldemoura@f5.com is_multipart = True 2971355St.nateldemoura@f5.com break 2981355St.nateldemoura@f5.com 2991355St.nateldemoura@f5.com if is_multipart: 3001355St.nateldemoura@f5.com body, content_type = self.multipart_encode(fields) 3011355St.nateldemoura@f5.com 3021355St.nateldemoura@f5.com else: 3031355St.nateldemoura@f5.com body, content_type = self.form_url_encode(fields) 3041355St.nateldemoura@f5.com 3051355St.nateldemoura@f5.com return body, content_type 3061355St.nateldemoura@f5.com 3071355St.nateldemoura@f5.com def form_url_encode(self, fields): 3081355St.nateldemoura@f5.com data = "&".join("%s=%s" % (name, value) 3091355St.nateldemoura@f5.com for name, value in fields.items()).encode() 3101355St.nateldemoura@f5.com return data, 'application/x-www-form-urlencoded' 3111355St.nateldemoura@f5.com 3121355St.nateldemoura@f5.com def multipart_encode(self, fields): 3131355St.nateldemoura@f5.com boundary = binascii.hexlify(os.urandom(16)).decode('ascii') 3141355St.nateldemoura@f5.com 3151355St.nateldemoura@f5.com body = '' 3161355St.nateldemoura@f5.com 3171355St.nateldemoura@f5.com for field, value in fields.items(): 3181355St.nateldemoura@f5.com filename = '' 3191355St.nateldemoura@f5.com datatype = '' 3201355St.nateldemoura@f5.com 3211355St.nateldemoura@f5.com if isinstance(value, dict): 3221355St.nateldemoura@f5.com datatype = 'text/plain' 3231355St.nateldemoura@f5.com filename = value['filename'] 3241355St.nateldemoura@f5.com 3251355St.nateldemoura@f5.com if value.get('type'): 3261355St.nateldemoura@f5.com datatype = value['type'] 3271355St.nateldemoura@f5.com 3281355St.nateldemoura@f5.com if not isinstance(value['data'], io.IOBase): 3291596Szelenkov@nginx.com pytest.fail('multipart encoding of file requires a stream.') 3301355St.nateldemoura@f5.com 3311355St.nateldemoura@f5.com data = value['data'].read() 3321355St.nateldemoura@f5.com 3331355St.nateldemoura@f5.com elif isinstance(value, str): 3341355St.nateldemoura@f5.com data = value 3351355St.nateldemoura@f5.com 3361355St.nateldemoura@f5.com else: 3371596Szelenkov@nginx.com pytest.fail('multipart requires a string or stream data') 3381355St.nateldemoura@f5.com 3391355St.nateldemoura@f5.com body += ( 3401355St.nateldemoura@f5.com "--%s\r\nContent-Disposition: form-data; name=\"%s\"" 3411355St.nateldemoura@f5.com ) % (boundary, field) 3421355St.nateldemoura@f5.com 3431355St.nateldemoura@f5.com if filename != '': 3441355St.nateldemoura@f5.com body += "; filename=\"%s\"" % filename 3451355St.nateldemoura@f5.com 3461355St.nateldemoura@f5.com body += "\r\n" 3471355St.nateldemoura@f5.com 3481355St.nateldemoura@f5.com if datatype != '': 3491355St.nateldemoura@f5.com body += "Content-Type: %s\r\n" % datatype 3501355St.nateldemoura@f5.com 3511355St.nateldemoura@f5.com body += "\r\n%s\r\n" % data 3521355St.nateldemoura@f5.com 3531355St.nateldemoura@f5.com body += "--%s--\r\n" % boundary 3541355St.nateldemoura@f5.com 3551355St.nateldemoura@f5.com return body.encode(), "multipart/form-data; boundary=%s" % boundary 356