11355St.nateldemoura@f5.comimport binascii 21355St.nateldemoura@f5.comimport io 31477Szelenkov@nginx.comimport json 41355St.nateldemoura@f5.comimport os 51019Szelenkov@nginx.comimport re 61477Szelenkov@nginx.comimport select 71477Szelenkov@nginx.comimport socket 81272Szelenkov@nginx.comimport time 91477Szelenkov@nginx.com 10*1635Szelenkov@nginx.comimport pytest 11*1635Szelenkov@nginx.comfrom conftest import option 121019Szelenkov@nginx.comfrom unit.main import TestUnit 131019Szelenkov@nginx.com 141019Szelenkov@nginx.com 151019Szelenkov@nginx.comclass TestHTTP(TestUnit): 161019Szelenkov@nginx.com def http(self, start_str, **kwargs): 171608Smax.romanov@nginx.com sock_type = kwargs.get('sock_type', 'ipv4') 181608Smax.romanov@nginx.com port = kwargs.get('port', 7080) 191608Smax.romanov@nginx.com url = kwargs.get('url', '/') 201019Szelenkov@nginx.com http = 'HTTP/1.0' if 'http_10' in kwargs else 'HTTP/1.1' 211019Szelenkov@nginx.com 221608Smax.romanov@nginx.com headers = kwargs.get('headers', 231608Smax.romanov@nginx.com {'Host': 'localhost', 'Connection': 'close'}) 241019Szelenkov@nginx.com 251608Smax.romanov@nginx.com body = kwargs.get('body', b'') 261019Szelenkov@nginx.com crlf = '\r\n' 271019Szelenkov@nginx.com 281019Szelenkov@nginx.com if 'addr' not in kwargs: 291019Szelenkov@nginx.com addr = '::1' if sock_type == 'ipv6' else '127.0.0.1' 301019Szelenkov@nginx.com else: 311019Szelenkov@nginx.com addr = kwargs['addr'] 321019Szelenkov@nginx.com 331019Szelenkov@nginx.com sock_types = { 341019Szelenkov@nginx.com 'ipv4': socket.AF_INET, 351019Szelenkov@nginx.com 'ipv6': socket.AF_INET6, 361019Szelenkov@nginx.com 'unix': socket.AF_UNIX, 371019Szelenkov@nginx.com } 381019Szelenkov@nginx.com 391019Szelenkov@nginx.com if 'sock' not in kwargs: 401019Szelenkov@nginx.com sock = socket.socket(sock_types[sock_type], socket.SOCK_STREAM) 411019Szelenkov@nginx.com 421019Szelenkov@nginx.com if ( 431019Szelenkov@nginx.com sock_type == sock_types['ipv4'] 441019Szelenkov@nginx.com or sock_type == sock_types['ipv6'] 451019Szelenkov@nginx.com ): 461019Szelenkov@nginx.com sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 471019Szelenkov@nginx.com 481019Szelenkov@nginx.com if 'wrapper' in kwargs: 491019Szelenkov@nginx.com sock = kwargs['wrapper'](sock) 501019Szelenkov@nginx.com 511019Szelenkov@nginx.com connect_args = addr if sock_type == 'unix' else (addr, port) 521019Szelenkov@nginx.com try: 531019Szelenkov@nginx.com sock.connect(connect_args) 541019Szelenkov@nginx.com except ConnectionRefusedError: 551019Szelenkov@nginx.com sock.close() 561596Szelenkov@nginx.com pytest.fail('Client can\'t connect to the server.') 571019Szelenkov@nginx.com 581019Szelenkov@nginx.com else: 591019Szelenkov@nginx.com sock = kwargs['sock'] 601019Szelenkov@nginx.com 611019Szelenkov@nginx.com if 'raw' not in kwargs: 621019Szelenkov@nginx.com req = ' '.join([start_str, url, http]) + crlf 631019Szelenkov@nginx.com 641256Szelenkov@nginx.com if body != b'': 651019Szelenkov@nginx.com if isinstance(body, str): 661019Szelenkov@nginx.com body = body.encode() 671355St.nateldemoura@f5.com elif isinstance(body, dict): 681355St.nateldemoura@f5.com body, content_type = self.form_encode(body) 691355St.nateldemoura@f5.com 701355St.nateldemoura@f5.com headers['Content-Type'] = content_type 711019Szelenkov@nginx.com 721019Szelenkov@nginx.com if 'Content-Length' not in headers: 731019Szelenkov@nginx.com headers['Content-Length'] = len(body) 741019Szelenkov@nginx.com 751019Szelenkov@nginx.com for header, value in headers.items(): 761019Szelenkov@nginx.com if isinstance(value, list): 771019Szelenkov@nginx.com for v in value: 781019Szelenkov@nginx.com req += header + ': ' + str(v) + crlf 791019Szelenkov@nginx.com 801019Szelenkov@nginx.com else: 811019Szelenkov@nginx.com req += header + ': ' + str(value) + crlf 821019Szelenkov@nginx.com 831019Szelenkov@nginx.com req = (req + crlf).encode() + body 841019Szelenkov@nginx.com 851019Szelenkov@nginx.com else: 861019Szelenkov@nginx.com req = start_str 871019Szelenkov@nginx.com 881019Szelenkov@nginx.com sock.sendall(req) 891019Szelenkov@nginx.com 901608Smax.romanov@nginx.com encoding = kwargs.get('encoding', 'utf-8') 911295St.nateldemoura@f5.com 921372Szelenkov@nginx.com self.log_out(req, encoding) 931019Szelenkov@nginx.com 941019Szelenkov@nginx.com resp = '' 951019Szelenkov@nginx.com 961019Szelenkov@nginx.com if 'no_recv' not in kwargs: 971424Szelenkov@nginx.com recvall_kwargs = {} 981424Szelenkov@nginx.com 991424Szelenkov@nginx.com if 'read_timeout' in kwargs: 1001424Szelenkov@nginx.com recvall_kwargs['read_timeout'] = kwargs['read_timeout'] 1011424Szelenkov@nginx.com 1021424Szelenkov@nginx.com if 'read_buffer_size' in kwargs: 1031424Szelenkov@nginx.com recvall_kwargs['buff_size'] = kwargs['read_buffer_size'] 1041424Szelenkov@nginx.com 1051424Szelenkov@nginx.com resp = self.recvall(sock, **recvall_kwargs).decode(encoding) 1061019Szelenkov@nginx.com 1071372Szelenkov@nginx.com self.log_in(resp) 1081019Szelenkov@nginx.com 1091019Szelenkov@nginx.com if 'raw_resp' not in kwargs: 1101019Szelenkov@nginx.com resp = self._resp_to_dict(resp) 1111019Szelenkov@nginx.com 1121295St.nateldemoura@f5.com headers = resp.get('headers') 1131295St.nateldemoura@f5.com if headers and headers.get('Transfer-Encoding') == 'chunked': 1141295St.nateldemoura@f5.com resp['body'] = self._parse_chunked_body(resp['body']).decode( 1151295St.nateldemoura@f5.com encoding 1161295St.nateldemoura@f5.com ) 1171295St.nateldemoura@f5.com 1181296St.nateldemoura@f5.com if 'json' in kwargs: 1191296St.nateldemoura@f5.com resp = self._parse_json(resp) 1201296St.nateldemoura@f5.com 1211019Szelenkov@nginx.com if 'start' not in kwargs: 1221019Szelenkov@nginx.com sock.close() 1231019Szelenkov@nginx.com return resp 1241019Szelenkov@nginx.com 1251019Szelenkov@nginx.com return (resp, sock) 1261019Szelenkov@nginx.com 1271372Szelenkov@nginx.com def log_out(self, log, encoding): 1281596Szelenkov@nginx.com if option.detailed: 1291372Szelenkov@nginx.com print('>>>') 1301372Szelenkov@nginx.com log = self.log_truncate(log) 1311372Szelenkov@nginx.com try: 1321372Szelenkov@nginx.com print(log.decode(encoding, 'ignore')) 1331372Szelenkov@nginx.com except UnicodeEncodeError: 1341372Szelenkov@nginx.com print(log) 1351372Szelenkov@nginx.com 1361372Szelenkov@nginx.com def log_in(self, log): 1371596Szelenkov@nginx.com if option.detailed: 1381372Szelenkov@nginx.com print('<<<') 1391372Szelenkov@nginx.com log = self.log_truncate(log) 1401372Szelenkov@nginx.com try: 1411372Szelenkov@nginx.com print(log) 1421372Szelenkov@nginx.com except UnicodeEncodeError: 1431372Szelenkov@nginx.com print(log.encode()) 1441372Szelenkov@nginx.com 1451372Szelenkov@nginx.com def log_truncate(self, log, limit=1024): 1461372Szelenkov@nginx.com len_log = len(log) 1471372Szelenkov@nginx.com if len_log > limit: 1481372Szelenkov@nginx.com log = log[:limit] 1491372Szelenkov@nginx.com appendix = '(...logged %s of %s bytes)' % (limit, len_log) 1501372Szelenkov@nginx.com 1511372Szelenkov@nginx.com if isinstance(log, bytes): 1521372Szelenkov@nginx.com appendix = appendix.encode() 1531372Szelenkov@nginx.com 1541372Szelenkov@nginx.com log = log + appendix 1551372Szelenkov@nginx.com 1561372Szelenkov@nginx.com return log 1571372Szelenkov@nginx.com 1581019Szelenkov@nginx.com def delete(self, **kwargs): 1591019Szelenkov@nginx.com return self.http('DELETE', **kwargs) 1601019Szelenkov@nginx.com 1611019Szelenkov@nginx.com def get(self, **kwargs): 1621019Szelenkov@nginx.com return self.http('GET', **kwargs) 1631019Szelenkov@nginx.com 1641172Szelenkov@nginx.com def head(self, **kwargs): 1651172Szelenkov@nginx.com return self.http('HEAD', **kwargs) 1661172Szelenkov@nginx.com 1671019Szelenkov@nginx.com def post(self, **kwargs): 1681019Szelenkov@nginx.com return self.http('POST', **kwargs) 1691019Szelenkov@nginx.com 1701019Szelenkov@nginx.com def put(self, **kwargs): 1711019Szelenkov@nginx.com return self.http('PUT', **kwargs) 1721019Szelenkov@nginx.com 1731424Szelenkov@nginx.com def recvall(self, sock, **kwargs): 1741444Szelenkov@nginx.com timeout_default = 60 1751444Szelenkov@nginx.com 1761608Smax.romanov@nginx.com timeout = kwargs.get('read_timeout', timeout_default) 1771608Smax.romanov@nginx.com buff_size = kwargs.get('buff_size', 4096) 1781424Szelenkov@nginx.com 1791019Szelenkov@nginx.com data = b'' 1801444Szelenkov@nginx.com while True: 1811444Szelenkov@nginx.com rlist = select.select([sock], [], [], timeout)[0] 1821444Szelenkov@nginx.com if not rlist: 1831444Szelenkov@nginx.com # For all current cases if the "read_timeout" was changed 1841444Szelenkov@nginx.com # than test do not expect to get a response from server. 1851444Szelenkov@nginx.com if timeout == timeout_default: 1861596Szelenkov@nginx.com pytest.fail('Can\'t read response from server.') 1871444Szelenkov@nginx.com break 1881444Szelenkov@nginx.com 1891019Szelenkov@nginx.com try: 1901019Szelenkov@nginx.com part = sock.recv(buff_size) 1911019Szelenkov@nginx.com except: 1921019Szelenkov@nginx.com break 1931019Szelenkov@nginx.com 1941019Szelenkov@nginx.com data += part 1951019Szelenkov@nginx.com 1961019Szelenkov@nginx.com if not len(part): 1971019Szelenkov@nginx.com break 1981019Szelenkov@nginx.com 1991019Szelenkov@nginx.com return data 2001019Szelenkov@nginx.com 2011019Szelenkov@nginx.com def _resp_to_dict(self, resp): 2021295St.nateldemoura@f5.com m = re.search(r'(.*?\x0d\x0a?)\x0d\x0a?(.*)', resp, re.M | re.S) 2031019Szelenkov@nginx.com 2041019Szelenkov@nginx.com if not m: 2051019Szelenkov@nginx.com return {} 2061019Szelenkov@nginx.com 2071019Szelenkov@nginx.com headers_text, body = m.group(1), m.group(2) 2081019Szelenkov@nginx.com 2091019Szelenkov@nginx.com p = re.compile('(.*?)\x0d\x0a?', re.M | re.S) 2101019Szelenkov@nginx.com headers_lines = p.findall(headers_text) 2111019Szelenkov@nginx.com 2121019Szelenkov@nginx.com status = re.search( 2131295St.nateldemoura@f5.com r'^HTTP\/\d\.\d\s(\d+)|$', headers_lines.pop(0) 2141019Szelenkov@nginx.com ).group(1) 2151019Szelenkov@nginx.com 2161019Szelenkov@nginx.com headers = {} 2171019Szelenkov@nginx.com for line in headers_lines: 2181295St.nateldemoura@f5.com m = re.search(r'(.*)\:\s(.*)', line) 2191019Szelenkov@nginx.com 2201019Szelenkov@nginx.com if m.group(1) not in headers: 2211019Szelenkov@nginx.com headers[m.group(1)] = m.group(2) 2221019Szelenkov@nginx.com 2231019Szelenkov@nginx.com elif isinstance(headers[m.group(1)], list): 2241019Szelenkov@nginx.com headers[m.group(1)].append(m.group(2)) 2251019Szelenkov@nginx.com 2261019Szelenkov@nginx.com else: 2271019Szelenkov@nginx.com headers[m.group(1)] = [headers[m.group(1)], m.group(2)] 2281019Szelenkov@nginx.com 2291019Szelenkov@nginx.com return {'status': int(status), 'headers': headers, 'body': body} 2301272Szelenkov@nginx.com 2311295St.nateldemoura@f5.com def _parse_chunked_body(self, raw_body): 2321295St.nateldemoura@f5.com if isinstance(raw_body, str): 2331295St.nateldemoura@f5.com raw_body = bytes(raw_body.encode()) 2341295St.nateldemoura@f5.com 2351295St.nateldemoura@f5.com crlf = b'\r\n' 2361295St.nateldemoura@f5.com chunks = raw_body.split(crlf) 2371295St.nateldemoura@f5.com 2381295St.nateldemoura@f5.com if len(chunks) < 3: 2391596Szelenkov@nginx.com pytest.fail('Invalid chunked body') 2401295St.nateldemoura@f5.com 2411295St.nateldemoura@f5.com if chunks.pop() != b'': 2421596Szelenkov@nginx.com pytest.fail('No CRLF at the end of the body') 2431295St.nateldemoura@f5.com 2441295St.nateldemoura@f5.com try: 2451295St.nateldemoura@f5.com last_size = int(chunks[-2], 16) 2461295St.nateldemoura@f5.com except: 2471596Szelenkov@nginx.com pytest.fail('Invalid zero size chunk') 2481295St.nateldemoura@f5.com 2491295St.nateldemoura@f5.com if last_size != 0 or chunks[-1] != b'': 2501596Szelenkov@nginx.com pytest.fail('Incomplete body') 2511295St.nateldemoura@f5.com 2521295St.nateldemoura@f5.com body = b'' 2531295St.nateldemoura@f5.com while len(chunks) >= 2: 2541295St.nateldemoura@f5.com try: 2551295St.nateldemoura@f5.com size = int(chunks.pop(0), 16) 2561295St.nateldemoura@f5.com except: 2571596Szelenkov@nginx.com pytest.fail('Invalid chunk size %s' % str(size)) 2581295St.nateldemoura@f5.com 2591295St.nateldemoura@f5.com if size == 0: 2601596Szelenkov@nginx.com assert len(chunks) == 1, 'last zero size' 2611295St.nateldemoura@f5.com break 2621295St.nateldemoura@f5.com 2631295St.nateldemoura@f5.com temp_body = crlf.join(chunks) 2641295St.nateldemoura@f5.com 2651295St.nateldemoura@f5.com body += temp_body[:size] 2661295St.nateldemoura@f5.com 2671295St.nateldemoura@f5.com temp_body = temp_body[size + len(crlf) :] 2681295St.nateldemoura@f5.com 2691295St.nateldemoura@f5.com chunks = temp_body.split(crlf) 2701295St.nateldemoura@f5.com 2711295St.nateldemoura@f5.com return body 2721295St.nateldemoura@f5.com 2731296St.nateldemoura@f5.com def _parse_json(self, resp): 2741296St.nateldemoura@f5.com headers = resp['headers'] 2751296St.nateldemoura@f5.com 2761596Szelenkov@nginx.com assert 'Content-Type' in headers 2771596Szelenkov@nginx.com assert headers['Content-Type'] == 'application/json' 2781296St.nateldemoura@f5.com 2791296St.nateldemoura@f5.com resp['body'] = json.loads(resp['body']) 2801296St.nateldemoura@f5.com 2811296St.nateldemoura@f5.com return resp 2821296St.nateldemoura@f5.com 2831296St.nateldemoura@f5.com def getjson(self, **kwargs): 2841296St.nateldemoura@f5.com return self.get(json=True, **kwargs) 2851296St.nateldemoura@f5.com 2861272Szelenkov@nginx.com def waitforsocket(self, port): 2871272Szelenkov@nginx.com ret = False 2881272Szelenkov@nginx.com 2891272Szelenkov@nginx.com for i in range(50): 2901272Szelenkov@nginx.com try: 2911272Szelenkov@nginx.com sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 2921272Szelenkov@nginx.com sock.connect(('127.0.0.1', port)) 2931272Szelenkov@nginx.com ret = True 2941272Szelenkov@nginx.com break 2951272Szelenkov@nginx.com except: 2961272Szelenkov@nginx.com sock.close() 2971272Szelenkov@nginx.com time.sleep(0.1) 2981272Szelenkov@nginx.com 2991272Szelenkov@nginx.com sock.close() 3001272Szelenkov@nginx.com 3011596Szelenkov@nginx.com assert ret, 'socket connected' 3021355St.nateldemoura@f5.com 3031355St.nateldemoura@f5.com def form_encode(self, fields): 3041355St.nateldemoura@f5.com is_multipart = False 3051355St.nateldemoura@f5.com 3061355St.nateldemoura@f5.com for _, value in fields.items(): 3071355St.nateldemoura@f5.com if isinstance(value, dict): 3081355St.nateldemoura@f5.com is_multipart = True 3091355St.nateldemoura@f5.com break 3101355St.nateldemoura@f5.com 3111355St.nateldemoura@f5.com if is_multipart: 3121355St.nateldemoura@f5.com body, content_type = self.multipart_encode(fields) 3131355St.nateldemoura@f5.com 3141355St.nateldemoura@f5.com else: 3151355St.nateldemoura@f5.com body, content_type = self.form_url_encode(fields) 3161355St.nateldemoura@f5.com 3171355St.nateldemoura@f5.com return body, content_type 3181355St.nateldemoura@f5.com 3191355St.nateldemoura@f5.com def form_url_encode(self, fields): 3201355St.nateldemoura@f5.com data = "&".join("%s=%s" % (name, value) 3211355St.nateldemoura@f5.com for name, value in fields.items()).encode() 3221355St.nateldemoura@f5.com return data, 'application/x-www-form-urlencoded' 3231355St.nateldemoura@f5.com 3241355St.nateldemoura@f5.com def multipart_encode(self, fields): 3251355St.nateldemoura@f5.com boundary = binascii.hexlify(os.urandom(16)).decode('ascii') 3261355St.nateldemoura@f5.com 3271355St.nateldemoura@f5.com body = '' 3281355St.nateldemoura@f5.com 3291355St.nateldemoura@f5.com for field, value in fields.items(): 3301355St.nateldemoura@f5.com filename = '' 3311355St.nateldemoura@f5.com datatype = '' 3321355St.nateldemoura@f5.com 3331355St.nateldemoura@f5.com if isinstance(value, dict): 3341355St.nateldemoura@f5.com datatype = 'text/plain' 3351355St.nateldemoura@f5.com filename = value['filename'] 3361355St.nateldemoura@f5.com 3371355St.nateldemoura@f5.com if value.get('type'): 3381355St.nateldemoura@f5.com datatype = value['type'] 3391355St.nateldemoura@f5.com 3401355St.nateldemoura@f5.com if not isinstance(value['data'], io.IOBase): 3411596Szelenkov@nginx.com pytest.fail('multipart encoding of file requires a stream.') 3421355St.nateldemoura@f5.com 3431355St.nateldemoura@f5.com data = value['data'].read() 3441355St.nateldemoura@f5.com 3451355St.nateldemoura@f5.com elif isinstance(value, str): 3461355St.nateldemoura@f5.com data = value 3471355St.nateldemoura@f5.com 3481355St.nateldemoura@f5.com else: 3491596Szelenkov@nginx.com pytest.fail('multipart requires a string or stream data') 3501355St.nateldemoura@f5.com 3511355St.nateldemoura@f5.com body += ( 3521355St.nateldemoura@f5.com "--%s\r\nContent-Disposition: form-data; name=\"%s\"" 3531355St.nateldemoura@f5.com ) % (boundary, field) 3541355St.nateldemoura@f5.com 3551355St.nateldemoura@f5.com if filename != '': 3561355St.nateldemoura@f5.com body += "; filename=\"%s\"" % filename 3571355St.nateldemoura@f5.com 3581355St.nateldemoura@f5.com body += "\r\n" 3591355St.nateldemoura@f5.com 3601355St.nateldemoura@f5.com if datatype != '': 3611355St.nateldemoura@f5.com body += "Content-Type: %s\r\n" % datatype 3621355St.nateldemoura@f5.com 3631355St.nateldemoura@f5.com body += "\r\n%s\r\n" % data 3641355St.nateldemoura@f5.com 3651355St.nateldemoura@f5.com body += "--%s--\r\n" % boundary 3661355St.nateldemoura@f5.com 3671355St.nateldemoura@f5.com return body.encode(), "multipart/form-data; boundary=%s" % boundary 368