1import grp 2import pwd 3import re 4import time 5 6import pytest 7from unit.applications.lang.python import TestApplicationPython 8from unit.option import option 9 10 11class TestPythonApplication(TestApplicationPython): 12 prerequisites = {'modules': {'python': 'all'}} 13 14 def findall(self, pattern): 15 with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: 16 return re.findall(pattern, f.read()) 17 18 def test_python_application_variables(self): 19 self.load('variables') 20 21 body = 'Test body string.' 22 23 resp = self.http( 24 b"""POST / HTTP/1.1 25Host: localhost 26Content-Length: %d 27Custom-Header: blah 28Custom-hEader: Blah 29Content-Type: text/html 30Connection: close 31custom-header: BLAH 32 33%s""" % (len(body), body.encode()), 34 raw=True, 35 ) 36 37 assert resp['status'] == 200, 'status' 38 headers = resp['headers'] 39 header_server = headers.pop('Server') 40 assert re.search(r'Unit/[\d\.]+', header_server), 'server header' 41 assert ( 42 headers.pop('Server-Software') == header_server 43 ), 'server software header' 44 45 date = headers.pop('Date') 46 assert date[-4:] == ' GMT', 'date header timezone' 47 assert ( 48 abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5 49 ), 'date header' 50 51 assert headers == { 52 'Connection': 'close', 53 'Content-Length': str(len(body)), 54 'Content-Type': 'text/html', 55 'Request-Method': 'POST', 56 'Request-Uri': '/', 57 'Http-Host': 'localhost', 58 'Server-Protocol': 'HTTP/1.1', 59 'Custom-Header': 'blah, Blah, BLAH', 60 'Wsgi-Version': '(1, 0)', 61 'Wsgi-Url-Scheme': 'http', 62 'Wsgi-Multithread': 'False', 63 'Wsgi-Multiprocess': 'True', 64 'Wsgi-Run-Once': 'False', 65 }, 'headers' 66 assert resp['body'] == body, 'body' 67 68 def test_python_application_query_string(self): 69 self.load('query_string') 70 71 resp = self.get(url='/?var1=val1&var2=val2') 72 73 assert ( 74 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 75 ), 'Query-String header' 76 77 def test_python_application_query_string_space(self): 78 self.load('query_string') 79 80 resp = self.get(url='/ ?var1=val1&var2=val2') 81 assert ( 82 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 83 ), 'Query-String space' 84 85 resp = self.get(url='/ %20?var1=val1&var2=val2') 86 assert ( 87 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 88 ), 'Query-String space 2' 89 90 resp = self.get(url='/ %20 ?var1=val1&var2=val2') 91 assert ( 92 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 93 ), 'Query-String space 3' 94 95 resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2') 96 assert ( 97 resp['headers']['Query-String'] == ' var1= val1 & var2=val2' 98 ), 'Query-String space 4' 99 100 def test_python_application_query_string_empty(self): 101 self.load('query_string') 102 103 resp = self.get(url='/?') 104 105 assert resp['status'] == 200, 'query string empty status' 106 assert resp['headers']['Query-String'] == '', 'query string empty' 107 108 def test_python_application_query_string_absent(self): 109 self.load('query_string') 110 111 resp = self.get() 112 113 assert resp['status'] == 200, 'query string absent status' 114 assert resp['headers']['Query-String'] == '', 'query string absent' 115 116 @pytest.mark.skip('not yet') 117 def test_python_application_server_port(self): 118 self.load('server_port') 119 120 assert ( 121 self.get()['headers']['Server-Port'] == '7080' 122 ), 'Server-Port header' 123 124 @pytest.mark.skip('not yet') 125 def test_python_application_working_directory_invalid(self): 126 self.load('empty') 127 128 assert 'success' in self.conf( 129 '"/blah"', 'applications/empty/working_directory' 130 ), 'configure invalid working_directory' 131 132 assert self.get()['status'] == 500, 'status' 133 134 def test_python_application_204_transfer_encoding(self): 135 self.load('204_no_content') 136 137 assert ( 138 'Transfer-Encoding' not in self.get()['headers'] 139 ), '204 header transfer encoding' 140 141 def test_python_application_ctx_iter_atexit(self): 142 self.load('ctx_iter_atexit') 143 144 resp = self.post( 145 headers={ 146 'Host': 'localhost', 147 'Connection': 'close', 148 'Content-Type': 'text/html', 149 }, 150 body='0123456789', 151 ) 152 153 assert resp['status'] == 200, 'ctx iter status' 154 assert resp['body'] == '0123456789', 'ctx iter body' 155 156 self.conf({"listeners": {}, "applications": {}}) 157 158 assert ( 159 self.wait_for_record(r'RuntimeError') is not None 160 ), 'ctx iter atexit' 161 162 def test_python_keepalive_body(self): 163 self.load('mirror') 164 165 assert self.get()['status'] == 200, 'init' 166 167 body = '0123456789' * 500 168 (resp, sock) = self.post( 169 headers={ 170 'Host': 'localhost', 171 'Connection': 'keep-alive', 172 'Content-Type': 'text/html', 173 }, 174 start=True, 175 body=body, 176 read_timeout=1, 177 ) 178 179 assert resp['body'] == body, 'keep-alive 1' 180 181 body = '0123456789' 182 resp = self.post( 183 headers={ 184 'Host': 'localhost', 185 'Connection': 'close', 186 'Content-Type': 'text/html', 187 }, 188 sock=sock, 189 body=body, 190 ) 191 192 assert resp['body'] == body, 'keep-alive 2' 193 194 def test_python_keepalive_reconfigure(self): 195 self.load('mirror') 196 197 assert self.get()['status'] == 200, 'init' 198 199 body = '0123456789' 200 conns = 3 201 socks = [] 202 203 for i in range(conns): 204 (resp, sock) = self.post( 205 headers={ 206 'Host': 'localhost', 207 'Connection': 'keep-alive', 208 'Content-Type': 'text/html', 209 }, 210 start=True, 211 body=body, 212 read_timeout=1, 213 ) 214 215 assert resp['body'] == body, 'keep-alive open' 216 217 self.load('mirror', processes=i + 1) 218 219 socks.append(sock) 220 221 for i in range(conns): 222 (resp, sock) = self.post( 223 headers={ 224 'Host': 'localhost', 225 'Connection': 'keep-alive', 226 'Content-Type': 'text/html', 227 }, 228 start=True, 229 sock=socks[i], 230 body=body, 231 read_timeout=1, 232 ) 233 234 assert resp['body'] == body, 'keep-alive request' 235 236 self.load('mirror', processes=i + 1) 237 238 for i in range(conns): 239 resp = self.post( 240 headers={ 241 'Host': 'localhost', 242 'Connection': 'close', 243 'Content-Type': 'text/html', 244 }, 245 sock=socks[i], 246 body=body, 247 ) 248 249 assert resp['body'] == body, 'keep-alive close' 250 251 self.load('mirror', processes=i + 1) 252 253 def test_python_keepalive_reconfigure_2(self): 254 self.load('mirror') 255 256 assert self.get()['status'] == 200, 'init' 257 258 body = '0123456789' 259 260 (resp, sock) = self.post( 261 headers={ 262 'Host': 'localhost', 263 'Connection': 'keep-alive', 264 'Content-Type': 'text/html', 265 }, 266 start=True, 267 body=body, 268 read_timeout=1, 269 ) 270 271 assert resp['body'] == body, 'reconfigure 2 keep-alive 1' 272 273 self.load('empty') 274 275 assert self.get()['status'] == 200, 'init' 276 277 (resp, sock) = self.post( 278 headers={ 279 'Host': 'localhost', 280 'Connection': 'close', 281 'Content-Type': 'text/html', 282 }, 283 start=True, 284 sock=sock, 285 body=body, 286 ) 287 288 assert resp['status'] == 200, 'reconfigure 2 keep-alive 2' 289 assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body' 290 291 assert 'success' in self.conf( 292 {"listeners": {}, "applications": {}} 293 ), 'reconfigure 2 clear configuration' 294 295 resp = self.get(sock=sock) 296 297 assert resp == {}, 'reconfigure 2 keep-alive 3' 298 299 def test_python_keepalive_reconfigure_3(self): 300 self.load('empty') 301 302 assert self.get()['status'] == 200, 'init' 303 304 (_, sock) = self.http( 305 b"""GET / HTTP/1.1 306""", 307 start=True, 308 raw=True, 309 no_recv=True, 310 ) 311 312 assert self.get()['status'] == 200 313 314 assert 'success' in self.conf( 315 {"listeners": {}, "applications": {}} 316 ), 'reconfigure 3 clear configuration' 317 318 resp = self.http( 319 b"""Host: localhost 320Connection: close 321 322""", 323 sock=sock, 324 raw=True, 325 ) 326 327 assert resp['status'] == 200, 'reconfigure 3' 328 329 def test_python_atexit(self): 330 self.load('atexit') 331 332 self.get() 333 334 self.conf({"listeners": {}, "applications": {}}) 335 336 assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' 337 338 def test_python_process_switch(self): 339 self.load('delayed', processes=2) 340 341 self.get( 342 headers={ 343 'Host': 'localhost', 344 'Content-Length': '0', 345 'X-Delay': '5', 346 'Connection': 'close', 347 }, 348 no_recv=True, 349 ) 350 351 headers_delay_1 = { 352 'Connection': 'close', 353 'Host': 'localhost', 354 'Content-Length': '0', 355 'X-Delay': '1', 356 } 357 358 self.get(headers=headers_delay_1, no_recv=True) 359 360 time.sleep(0.5) 361 362 for _ in range(10): 363 self.get(headers=headers_delay_1, no_recv=True) 364 365 self.get(headers=headers_delay_1) 366 367 @pytest.mark.skip('not yet') 368 def test_python_application_start_response_exit(self): 369 self.load('start_response_exit') 370 371 assert self.get()['status'] == 500, 'start response exit' 372 373 def test_python_application_input_iter(self): 374 self.load('input_iter') 375 376 body = '''0123456789 377next line 378 379last line''' 380 381 resp = self.post(body=body) 382 assert resp['body'] == body, 'input iter' 383 assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines' 384 385 def test_python_application_input_readline(self): 386 self.load('input_readline') 387 388 body = '''0123456789 389next line 390 391last line''' 392 393 resp = self.post(body=body) 394 assert resp['body'] == body, 'input readline' 395 assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines' 396 397 def test_python_application_input_readline_size(self): 398 self.load('input_readline_size') 399 400 body = '''0123456789 401next line 402 403last line''' 404 405 assert self.post(body=body)['body'] == body, 'input readline size' 406 assert ( 407 self.post(body='0123')['body'] == '0123' 408 ), 'input readline size less' 409 410 def test_python_application_input_readlines(self): 411 self.load('input_readlines') 412 413 body = '''0123456789 414next line 415 416last line''' 417 418 resp = self.post(body=body) 419 assert resp['body'] == body, 'input readlines' 420 assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines' 421 422 def test_python_application_input_readlines_huge(self): 423 self.load('input_readlines') 424 425 body = ( 426 '''0123456789 abcdefghi 427next line: 0123456789 abcdefghi 428 429last line: 987654321 430''' 431 * 512 432 ) 433 434 assert ( 435 self.post(body=body, read_buffer_size=16384)['body'] == body 436 ), 'input readlines huge' 437 438 def test_python_application_input_read_length(self): 439 self.load('input_read_length') 440 441 body = '0123456789' 442 443 resp = self.post( 444 headers={ 445 'Host': 'localhost', 446 'Input-Length': '5', 447 'Connection': 'close', 448 }, 449 body=body, 450 ) 451 452 assert resp['body'] == body[:5], 'input read length lt body' 453 454 resp = self.post( 455 headers={ 456 'Host': 'localhost', 457 'Input-Length': '15', 458 'Connection': 'close', 459 }, 460 body=body, 461 ) 462 463 assert resp['body'] == body, 'input read length gt body' 464 465 resp = self.post( 466 headers={ 467 'Host': 'localhost', 468 'Input-Length': '0', 469 'Connection': 'close', 470 }, 471 body=body, 472 ) 473 474 assert resp['body'] == '', 'input read length zero' 475 476 resp = self.post( 477 headers={ 478 'Host': 'localhost', 479 'Input-Length': '-1', 480 'Connection': 'close', 481 }, 482 body=body, 483 ) 484 485 assert resp['body'] == body, 'input read length negative' 486 487 @pytest.mark.skip('not yet') 488 def test_python_application_errors_write(self): 489 self.load('errors_write') 490 491 self.get() 492 493 assert ( 494 self.wait_for_record(r'\[error\].+Error in application\.') 495 is not None 496 ), 'errors write' 497 498 def test_python_application_body_array(self): 499 self.load('body_array') 500 501 assert self.get()['body'] == '0123456789', 'body array' 502 503 def test_python_application_body_io(self): 504 self.load('body_io') 505 506 assert self.get()['body'] == '0123456789', 'body io' 507 508 def test_python_application_body_io_file(self): 509 self.load('body_io_file') 510 511 assert self.get()['body'] == 'body\n', 'body io file' 512 513 @pytest.mark.skip('not yet') 514 def test_python_application_syntax_error(self, skip_alert): 515 skip_alert(r'Python failed to import module "wsgi"') 516 self.load('syntax_error') 517 518 assert self.get()['status'] == 500, 'syntax error' 519 520 def test_python_application_loading_error(self, skip_alert): 521 skip_alert(r'Python failed to import module "blah"') 522 523 self.load('empty', module="blah") 524 525 assert self.get()['status'] == 503, 'loading error' 526 527 def test_python_application_close(self): 528 self.load('close') 529 530 self.get() 531 532 assert self.wait_for_record(r'Close called\.') is not None, 'close' 533 534 def test_python_application_close_error(self): 535 self.load('close_error') 536 537 self.get() 538 539 assert ( 540 self.wait_for_record(r'Close called\.') is not None 541 ), 'close error' 542 543 def test_python_application_not_iterable(self): 544 self.load('not_iterable') 545 546 self.get() 547 548 assert ( 549 self.wait_for_record( 550 r'\[error\].+the application returned not an iterable object' 551 ) 552 is not None 553 ), 'not iterable' 554 555 def test_python_application_write(self): 556 self.load('write') 557 558 assert self.get()['body'] == '0123456789', 'write' 559 560 def test_python_application_threading(self): 561 """wait_for_record() timeouts after 5s while every thread works at 562 least 3s. So without releasing GIL test should fail. 563 """ 564 565 self.load('threading') 566 567 for _ in range(10): 568 self.get(no_recv=True) 569 570 assert ( 571 self.wait_for_record(r'\(5\) Thread: 100') is not None 572 ), 'last thread finished' 573 574 def test_python_application_iter_exception(self): 575 self.load('iter_exception') 576 577 # Default request doesn't lead to the exception. 578 579 resp = self.get( 580 headers={ 581 'Host': 'localhost', 582 'X-Skip': '9', 583 'X-Chunked': '1', 584 'Connection': 'close', 585 } 586 ) 587 assert resp['status'] == 200, 'status' 588 assert resp['body'] == 'XXXXXXX', 'body' 589 590 # Exception before start_response(). 591 592 assert self.get()['status'] == 503, 'error' 593 594 assert self.wait_for_record(r'Traceback') is not None, 'traceback' 595 assert ( 596 self.wait_for_record(r'raise Exception\(\'first exception\'\)') 597 is not None 598 ), 'first exception raise' 599 assert len(self.findall(r'Traceback')) == 1, 'traceback count 1' 600 601 # Exception after start_response(), before first write(). 602 603 assert ( 604 self.get( 605 headers={ 606 'Host': 'localhost', 607 'X-Skip': '1', 608 'Connection': 'close', 609 } 610 )['status'] 611 == 503 612 ), 'error 2' 613 614 assert ( 615 self.wait_for_record(r'raise Exception\(\'second exception\'\)') 616 is not None 617 ), 'exception raise second' 618 assert len(self.findall(r'Traceback')) == 2, 'traceback count 2' 619 620 # Exception after first write(), before first __next__(). 621 622 _, sock = self.get( 623 headers={ 624 'Host': 'localhost', 625 'X-Skip': '2', 626 'Connection': 'keep-alive', 627 }, 628 start=True, 629 ) 630 631 assert ( 632 self.wait_for_record(r'raise Exception\(\'third exception\'\)') 633 is not None 634 ), 'exception raise third' 635 assert len(self.findall(r'Traceback')) == 3, 'traceback count 3' 636 637 assert self.get(sock=sock) == {}, 'closed connection' 638 639 # Exception after first write(), before first __next__(), 640 # chunked (incomplete body). 641 642 resp = self.get( 643 headers={ 644 'Host': 'localhost', 645 'X-Skip': '2', 646 'X-Chunked': '1', 647 'Connection': 'close', 648 }, 649 raw_resp=True, 650 ) 651 if resp: 652 assert resp[-5:] != '0\r\n\r\n', 'incomplete body' 653 assert len(self.findall(r'Traceback')) == 4, 'traceback count 4' 654 655 # Exception in __next__(). 656 657 _, sock = self.get( 658 headers={ 659 'Host': 'localhost', 660 'X-Skip': '3', 661 'Connection': 'keep-alive', 662 }, 663 start=True, 664 ) 665 666 assert ( 667 self.wait_for_record(r'raise Exception\(\'next exception\'\)') 668 is not None 669 ), 'exception raise next' 670 assert len(self.findall(r'Traceback')) == 5, 'traceback count 5' 671 672 assert self.get(sock=sock) == {}, 'closed connection 2' 673 674 # Exception in __next__(), chunked (incomplete body). 675 676 resp = self.get( 677 headers={ 678 'Host': 'localhost', 679 'X-Skip': '3', 680 'X-Chunked': '1', 681 'Connection': 'close', 682 }, 683 raw_resp=True, 684 ) 685 if resp: 686 assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2' 687 assert len(self.findall(r'Traceback')) == 6, 'traceback count 6' 688 689 # Exception before start_response() and in close(). 690 691 assert ( 692 self.get( 693 headers={ 694 'Host': 'localhost', 695 'X-Not-Skip-Close': '1', 696 'Connection': 'close', 697 } 698 )['status'] 699 == 503 700 ), 'error' 701 702 assert ( 703 self.wait_for_record(r'raise Exception\(\'close exception\'\)') 704 is not None 705 ), 'exception raise close' 706 assert len(self.findall(r'Traceback')) == 8, 'traceback count 8' 707 708 def test_python_user_group(self, is_su): 709 if not is_su: 710 pytest.skip('requires root') 711 712 nobody_uid = pwd.getpwnam('nobody').pw_uid 713 714 group = 'nobody' 715 716 try: 717 group_id = grp.getgrnam(group).gr_gid 718 except KeyError: 719 group = 'nogroup' 720 group_id = grp.getgrnam(group).gr_gid 721 722 self.load('user_group') 723 724 obj = self.getjson()['body'] 725 assert obj['UID'] == nobody_uid, 'nobody uid' 726 assert obj['GID'] == group_id, 'nobody gid' 727 728 self.load('user_group', user='nobody') 729 730 obj = self.getjson()['body'] 731 assert obj['UID'] == nobody_uid, 'nobody uid user=nobody' 732 assert obj['GID'] == group_id, 'nobody gid user=nobody' 733 734 self.load('user_group', user='nobody', group=group) 735 736 obj = self.getjson()['body'] 737 assert obj['UID'] == nobody_uid, ( 738 'nobody uid user=nobody group=%s' % group 739 ) 740 741 assert obj['GID'] == group_id, ( 742 'nobody gid user=nobody group=%s' % group 743 ) 744 745 self.load('user_group', group=group) 746 747 obj = self.getjson()['body'] 748 assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group 749 750 assert obj['GID'] == group_id, 'nobody gid group=%s' % group 751 752 self.load('user_group', user='root') 753 754 obj = self.getjson()['body'] 755 assert obj['UID'] == 0, 'root uid user=root' 756 assert obj['GID'] == 0, 'root gid user=root' 757 758 group = 'root' 759 760 try: 761 grp.getgrnam(group) 762 group = True 763 except KeyError: 764 group = False 765 766 if group: 767 self.load('user_group', user='root', group='root') 768 769 obj = self.getjson()['body'] 770 assert obj['UID'] == 0, 'root uid user=root group=root' 771 assert obj['GID'] == 0, 'root gid user=root group=root' 772 773 self.load('user_group', group='root') 774 775 obj = self.getjson()['body'] 776 assert obj['UID'] == nobody_uid, 'root uid group=root' 777 assert obj['GID'] == 0, 'root gid group=root' 778 779 def test_python_application_callable(self, skip_alert): 780 skip_alert(r'Python failed to get "blah" from module') 781 self.load('callable') 782 783 assert self.get()['status'] == 204, 'default application response' 784 785 self.load('callable', callable="app") 786 787 assert self.get()['status'] == 200, 'callable response' 788 789 self.load('callable', callable="blah") 790 791 assert self.get()['status'] not in [200, 204], 'callable response inv' 792 793 def test_python_application_threads(self): 794 self.load('threads', threads=4) 795 796 socks = [] 797 798 for i in range(4): 799 (_, sock) = self.get( 800 headers={ 801 'Host': 'localhost', 802 'X-Delay': '2', 803 'Connection': 'close', 804 }, 805 no_recv=True, 806 start=True, 807 ) 808 809 socks.append(sock) 810 811 threads = set() 812 813 for sock in socks: 814 resp = self.recvall(sock).decode('utf-8') 815 816 self.log_in(resp) 817 818 resp = self._resp_to_dict(resp) 819 820 assert resp['status'] == 200, 'status' 821 822 threads.add(resp['headers']['X-Thread']) 823 824 assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' 825 826 sock.close() 827 828 assert len(socks) == len(threads), 'threads differs' 829