1import grp 2import os 3import pwd 4import re 5import time 6 7import pytest 8 9from unit.applications.lang.python import TestApplicationPython 10 11 12class TestPythonApplication(TestApplicationPython): 13 prerequisites = {'modules': {'python': 'all'}} 14 15 def test_python_application_variables(self): 16 self.load('variables') 17 18 body = 'Test body string.' 19 20 resp = self.http( 21 b"""POST / HTTP/1.1 22Host: localhost 23Content-Length: %d 24Custom-Header: blah 25Custom-hEader: Blah 26Content-Type: text/html 27Connection: close 28custom-header: BLAH 29 30%s""" 31 % (len(body), body.encode()), 32 raw=True, 33 ) 34 35 assert resp['status'] == 200, 'status' 36 headers = resp['headers'] 37 header_server = headers.pop('Server') 38 assert re.search(r'Unit/[\d\.]+', header_server), 'server header' 39 assert ( 40 headers.pop('Server-Software') == header_server 41 ), 'server software header' 42 43 date = headers.pop('Date') 44 assert date[-4:] == ' GMT', 'date header timezone' 45 assert ( 46 abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5 47 ), 'date header' 48 49 assert headers == { 50 'Connection': 'close', 51 'Content-Length': str(len(body)), 52 'Content-Type': 'text/html', 53 'Request-Method': 'POST', 54 'Request-Uri': '/', 55 'Http-Host': 'localhost', 56 'Server-Protocol': 'HTTP/1.1', 57 'Custom-Header': 'blah, Blah, BLAH', 58 'Wsgi-Version': '(1, 0)', 59 'Wsgi-Url-Scheme': 'http', 60 'Wsgi-Multithread': 'False', 61 'Wsgi-Multiprocess': 'True', 62 'Wsgi-Run-Once': 'False', 63 }, 'headers' 64 assert resp['body'] == body, 'body' 65 66 def test_python_application_query_string(self): 67 self.load('query_string') 68 69 resp = self.get(url='/?var1=val1&var2=val2') 70 71 assert ( 72 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 73 ), 'Query-String header' 74 75 def test_python_application_query_string_space(self): 76 self.load('query_string') 77 78 resp = self.get(url='/ ?var1=val1&var2=val2') 79 assert ( 80 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 81 ), 'Query-String space' 82 83 resp = self.get(url='/ %20?var1=val1&var2=val2') 84 assert ( 85 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 86 ), 'Query-String space 2' 87 88 resp = self.get(url='/ %20 ?var1=val1&var2=val2') 89 assert ( 90 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 91 ), 'Query-String space 3' 92 93 resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2') 94 assert ( 95 resp['headers']['Query-String'] == ' var1= val1 & var2=val2' 96 ), 'Query-String space 4' 97 98 def test_python_application_query_string_empty(self): 99 self.load('query_string') 100 101 resp = self.get(url='/?') 102 103 assert resp['status'] == 200, 'query string empty status' 104 assert resp['headers']['Query-String'] == '', 'query string empty' 105 106 def test_python_application_query_string_absent(self): 107 self.load('query_string') 108 109 resp = self.get() 110 111 assert resp['status'] == 200, 'query string absent status' 112 assert resp['headers']['Query-String'] == '', 'query string absent' 113 114 @pytest.mark.skip('not yet') 115 def test_python_application_server_port(self): 116 self.load('server_port') 117 118 assert ( 119 self.get()['headers']['Server-Port'] == '7080' 120 ), 'Server-Port header' 121 122 @pytest.mark.skip('not yet') 123 def test_python_application_working_directory_invalid(self): 124 self.load('empty') 125 126 assert 'success' in self.conf( 127 '"/blah"', 'applications/empty/working_directory' 128 ), 'configure invalid working_directory' 129 130 assert self.get()['status'] == 500, 'status' 131 132 def test_python_application_204_transfer_encoding(self): 133 self.load('204_no_content') 134 135 assert ( 136 'Transfer-Encoding' not in self.get()['headers'] 137 ), '204 header transfer encoding' 138 139 def test_python_application_ctx_iter_atexit(self): 140 self.load('ctx_iter_atexit') 141 142 resp = self.post( 143 headers={ 144 'Host': 'localhost', 145 'Connection': 'close', 146 'Content-Type': 'text/html', 147 }, 148 body='0123456789', 149 ) 150 151 assert resp['status'] == 200, 'ctx iter status' 152 assert resp['body'] == '0123456789', 'ctx iter body' 153 154 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 155 156 assert ( 157 self.wait_for_record(r'RuntimeError') is not None 158 ), 'ctx iter atexit' 159 160 def test_python_keepalive_body(self): 161 self.load('mirror') 162 163 assert self.get()['status'] == 200, 'init' 164 165 body = '0123456789' * 500 166 (resp, sock) = self.post( 167 headers={ 168 'Host': 'localhost', 169 'Connection': 'keep-alive', 170 'Content-Type': 'text/html', 171 }, 172 start=True, 173 body=body, 174 read_timeout=1, 175 ) 176 177 assert resp['body'] == body, 'keep-alive 1' 178 179 body = '0123456789' 180 resp = self.post( 181 headers={ 182 'Host': 'localhost', 183 'Connection': 'close', 184 'Content-Type': 'text/html', 185 }, 186 sock=sock, 187 body=body, 188 ) 189 190 assert resp['body'] == body, 'keep-alive 2' 191 192 def test_python_keepalive_reconfigure(self): 193 self.load('mirror') 194 195 assert self.get()['status'] == 200, 'init' 196 197 body = '0123456789' 198 conns = 3 199 socks = [] 200 201 for i in range(conns): 202 (resp, sock) = self.post( 203 headers={ 204 'Host': 'localhost', 205 'Connection': 'keep-alive', 206 'Content-Type': 'text/html', 207 }, 208 start=True, 209 body=body, 210 read_timeout=1, 211 ) 212 213 assert resp['body'] == body, 'keep-alive open' 214 215 self.load('mirror', processes=i + 1) 216 217 socks.append(sock) 218 219 for i in range(conns): 220 (resp, sock) = self.post( 221 headers={ 222 'Host': 'localhost', 223 'Connection': 'keep-alive', 224 'Content-Type': 'text/html', 225 }, 226 start=True, 227 sock=socks[i], 228 body=body, 229 read_timeout=1, 230 ) 231 232 assert resp['body'] == body, 'keep-alive request' 233 234 self.load('mirror', processes=i + 1) 235 236 for i in range(conns): 237 resp = self.post( 238 headers={ 239 'Host': 'localhost', 240 'Connection': 'close', 241 'Content-Type': 'text/html', 242 }, 243 sock=socks[i], 244 body=body, 245 ) 246 247 assert resp['body'] == body, 'keep-alive close' 248 249 self.load('mirror', processes=i + 1) 250 251 def test_python_keepalive_reconfigure_2(self): 252 self.load('mirror') 253 254 assert self.get()['status'] == 200, 'init' 255 256 body = '0123456789' 257 258 (resp, sock) = self.post( 259 headers={ 260 'Host': 'localhost', 261 'Connection': 'keep-alive', 262 'Content-Type': 'text/html', 263 }, 264 start=True, 265 body=body, 266 read_timeout=1, 267 ) 268 269 assert resp['body'] == body, 'reconfigure 2 keep-alive 1' 270 271 self.load('empty') 272 273 assert self.get()['status'] == 200, 'init' 274 275 (resp, sock) = self.post( 276 headers={ 277 'Host': 'localhost', 278 'Connection': 'close', 279 'Content-Type': 'text/html', 280 }, 281 start=True, 282 sock=sock, 283 body=body, 284 ) 285 286 assert resp['status'] == 200, 'reconfigure 2 keep-alive 2' 287 assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body' 288 289 assert 'success' in self.conf( 290 {"listeners": {}, "applications": {}} 291 ), 'reconfigure 2 clear configuration' 292 293 resp = self.get(sock=sock) 294 295 assert resp == {}, 'reconfigure 2 keep-alive 3' 296 297 def test_python_keepalive_reconfigure_3(self): 298 self.load('empty') 299 300 assert self.get()['status'] == 200, 'init' 301 302 (_, sock) = self.http( 303 b"""GET / HTTP/1.1 304""", 305 start=True, 306 raw=True, 307 no_recv=True, 308 ) 309 310 assert self.get()['status'] == 200 311 312 assert 'success' in self.conf( 313 {"listeners": {}, "applications": {}} 314 ), 'reconfigure 3 clear configuration' 315 316 resp = self.http( 317 b"""Host: localhost 318Connection: close 319 320""", 321 sock=sock, 322 raw=True, 323 ) 324 325 assert resp['status'] == 200, 'reconfigure 3' 326 327 def test_python_atexit(self): 328 self.load('atexit') 329 330 self.get() 331 332 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 333 334 assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' 335 336 def test_python_process_switch(self): 337 self.load('delayed', processes=2) 338 339 self.get( 340 headers={ 341 'Host': 'localhost', 342 'Content-Length': '0', 343 'X-Delay': '5', 344 'Connection': 'close', 345 }, 346 no_recv=True, 347 ) 348 349 headers_delay_1 = { 350 'Connection': 'close', 351 'Host': 'localhost', 352 'Content-Length': '0', 353 'X-Delay': '1', 354 } 355 356 self.get(headers=headers_delay_1, no_recv=True) 357 358 time.sleep(0.5) 359 360 for _ in range(10): 361 self.get(headers=headers_delay_1, no_recv=True) 362 363 self.get(headers=headers_delay_1) 364 365 @pytest.mark.skip('not yet') 366 def test_python_application_start_response_exit(self): 367 self.load('start_response_exit') 368 369 assert self.get()['status'] == 500, 'start response exit' 370 371 def test_python_application_input_iter(self): 372 self.load('input_iter') 373 374 body = '''0123456789 375next line 376 377last line''' 378 379 resp = self.post(body=body) 380 assert resp['body'] == body, 'input iter' 381 assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines' 382 383 def test_python_application_input_readline(self): 384 self.load('input_readline') 385 386 body = '''0123456789 387next line 388 389last line''' 390 391 resp = self.post(body=body) 392 assert resp['body'] == body, 'input readline' 393 assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines' 394 395 def test_python_application_input_readline_size(self): 396 self.load('input_readline_size') 397 398 body = '''0123456789 399next line 400 401last line''' 402 403 assert self.post(body=body)['body'] == body, 'input readline size' 404 assert ( 405 self.post(body='0123')['body'] == '0123' 406 ), 'input readline size less' 407 408 def test_python_application_input_readlines(self): 409 self.load('input_readlines') 410 411 body = '''0123456789 412next line 413 414last line''' 415 416 resp = self.post(body=body) 417 assert resp['body'] == body, 'input readlines' 418 assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines' 419 420 def test_python_application_input_readlines_huge(self): 421 self.load('input_readlines') 422 423 body = ( 424 '''0123456789 abcdefghi 425next line: 0123456789 abcdefghi 426 427last line: 987654321 428''' 429 * 512 430 ) 431 432 assert ( 433 self.post(body=body, read_buffer_size=16384)['body'] == body 434 ), 'input readlines huge' 435 436 def test_python_application_input_read_length(self): 437 self.load('input_read_length') 438 439 body = '0123456789' 440 441 resp = self.post( 442 headers={ 443 'Host': 'localhost', 444 'Input-Length': '5', 445 'Connection': 'close', 446 }, 447 body=body, 448 ) 449 450 assert resp['body'] == body[:5], 'input read length lt body' 451 452 resp = self.post( 453 headers={ 454 'Host': 'localhost', 455 'Input-Length': '15', 456 'Connection': 'close', 457 }, 458 body=body, 459 ) 460 461 assert resp['body'] == body, 'input read length gt body' 462 463 resp = self.post( 464 headers={ 465 'Host': 'localhost', 466 'Input-Length': '0', 467 'Connection': 'close', 468 }, 469 body=body, 470 ) 471 472 assert resp['body'] == '', 'input read length zero' 473 474 resp = self.post( 475 headers={ 476 'Host': 'localhost', 477 'Input-Length': '-1', 478 'Connection': 'close', 479 }, 480 body=body, 481 ) 482 483 assert resp['body'] == body, 'input read length negative' 484 485 @pytest.mark.skip('not yet') 486 def test_python_application_errors_write(self): 487 self.load('errors_write') 488 489 self.get() 490 491 assert ( 492 self.wait_for_record(r'\[error\].+Error in application\.') 493 is not None 494 ), 'errors write' 495 496 def test_python_application_body_array(self): 497 self.load('body_array') 498 499 assert self.get()['body'] == '0123456789', 'body array' 500 501 def test_python_application_body_io(self): 502 self.load('body_io') 503 504 assert self.get()['body'] == '0123456789', 'body io' 505 506 def test_python_application_body_io_file(self): 507 self.load('body_io_file') 508 509 assert self.get()['body'] == 'body\n', 'body io file' 510 511 @pytest.mark.skip('not yet') 512 def test_python_application_syntax_error(self, skip_alert): 513 skip_alert(r'Python failed to import module "wsgi"') 514 self.load('syntax_error') 515 516 assert self.get()['status'] == 500, 'syntax error' 517 518 def test_python_application_loading_error(self, skip_alert): 519 skip_alert(r'Python failed to import module "blah"') 520 521 self.load('empty', module="blah") 522 523 assert self.get()['status'] == 503, 'loading error' 524 525 def test_python_application_close(self): 526 self.load('close') 527 528 self.get() 529 530 assert self.wait_for_record(r'Close called\.') is not None, 'close' 531 532 def test_python_application_close_error(self): 533 self.load('close_error') 534 535 self.get() 536 537 assert ( 538 self.wait_for_record(r'Close called\.') is not None 539 ), 'close error' 540 541 def test_python_application_not_iterable(self): 542 self.load('not_iterable') 543 544 self.get() 545 546 assert ( 547 self.wait_for_record( 548 r'\[error\].+the application returned not an iterable object' 549 ) 550 is not None 551 ), 'not iterable' 552 553 def test_python_application_write(self): 554 self.load('write') 555 556 assert self.get()['body'] == '0123456789', 'write' 557 558 def test_python_application_threading(self): 559 """wait_for_record() timeouts after 5s while every thread works at 560 least 3s. So without releasing GIL test should fail. 561 """ 562 563 self.load('threading') 564 565 for _ in range(10): 566 self.get(no_recv=True) 567 568 assert ( 569 self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None 570 ), 'last thread finished' 571 572 def test_python_application_iter_exception(self): 573 self.load('iter_exception') 574 575 # Default request doesn't lead to the exception. 576 577 resp = self.get( 578 headers={ 579 'Host': 'localhost', 580 'X-Skip': '9', 581 'X-Chunked': '1', 582 'Connection': 'close', 583 } 584 ) 585 assert resp['status'] == 200, 'status' 586 assert resp['body'] == 'XXXXXXX', 'body' 587 588 # Exception before start_response(). 589 590 assert self.get()['status'] == 503, 'error' 591 592 assert self.wait_for_record(r'Traceback') is not None, 'traceback' 593 assert ( 594 self.wait_for_record(r'raise Exception\(\'first exception\'\)') 595 is not None 596 ), 'first exception raise' 597 assert len(self.findall(r'Traceback')) == 1, 'traceback count 1' 598 599 # Exception after start_response(), before first write(). 600 601 assert ( 602 self.get( 603 headers={ 604 'Host': 'localhost', 605 'X-Skip': '1', 606 'Connection': 'close', 607 } 608 )['status'] 609 == 503 610 ), 'error 2' 611 612 assert ( 613 self.wait_for_record(r'raise Exception\(\'second exception\'\)') 614 is not None 615 ), 'exception raise second' 616 assert len(self.findall(r'Traceback')) == 2, 'traceback count 2' 617 618 # Exception after first write(), before first __next__(). 619 620 _, sock = self.get( 621 headers={ 622 'Host': 'localhost', 623 'X-Skip': '2', 624 'Connection': 'keep-alive', 625 }, 626 start=True, 627 ) 628 629 assert ( 630 self.wait_for_record(r'raise Exception\(\'third exception\'\)') 631 is not None 632 ), 'exception raise third' 633 assert len(self.findall(r'Traceback')) == 3, 'traceback count 3' 634 635 assert self.get(sock=sock) == {}, 'closed connection' 636 637 # Exception after first write(), before first __next__(), 638 # chunked (incomplete body). 639 640 resp = self.get( 641 headers={ 642 'Host': 'localhost', 643 'X-Skip': '2', 644 'X-Chunked': '1', 645 'Connection': 'close', 646 }, 647 raw_resp=True, 648 ) 649 if resp: 650 assert resp[-5:] != '0\r\n\r\n', 'incomplete body' 651 assert len(self.findall(r'Traceback')) == 4, 'traceback count 4' 652 653 # Exception in __next__(). 654 655 _, sock = self.get( 656 headers={ 657 'Host': 'localhost', 658 'X-Skip': '3', 659 'Connection': 'keep-alive', 660 }, 661 start=True, 662 ) 663 664 assert ( 665 self.wait_for_record(r'raise Exception\(\'next exception\'\)') 666 is not None 667 ), 'exception raise next' 668 assert len(self.findall(r'Traceback')) == 5, 'traceback count 5' 669 670 assert self.get(sock=sock) == {}, 'closed connection 2' 671 672 # Exception in __next__(), chunked (incomplete body). 673 674 resp = self.get( 675 headers={ 676 'Host': 'localhost', 677 'X-Skip': '3', 678 'X-Chunked': '1', 679 'Connection': 'close', 680 }, 681 raw_resp=True, 682 ) 683 if resp: 684 assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2' 685 assert len(self.findall(r'Traceback')) == 6, 'traceback count 6' 686 687 # Exception before start_response() and in close(). 688 689 assert ( 690 self.get( 691 headers={ 692 'Host': 'localhost', 693 'X-Not-Skip-Close': '1', 694 'Connection': 'close', 695 } 696 )['status'] 697 == 503 698 ), 'error' 699 700 assert ( 701 self.wait_for_record(r'raise Exception\(\'close exception\'\)') 702 is not None 703 ), 'exception raise close' 704 assert len(self.findall(r'Traceback')) == 8, 'traceback count 8' 705 706 def test_python_user_group(self, is_su): 707 if not is_su: 708 pytest.skip('requires root') 709 710 nobody_uid = pwd.getpwnam('nobody').pw_uid 711 712 group = 'nobody' 713 714 try: 715 group_id = grp.getgrnam(group).gr_gid 716 except KeyError: 717 group = 'nogroup' 718 group_id = grp.getgrnam(group).gr_gid 719 720 self.load('user_group') 721 722 obj = self.getjson()['body'] 723 assert obj['UID'] == nobody_uid, 'nobody uid' 724 assert obj['GID'] == group_id, 'nobody gid' 725 726 self.load('user_group', user='nobody') 727 728 obj = self.getjson()['body'] 729 assert obj['UID'] == nobody_uid, 'nobody uid user=nobody' 730 assert obj['GID'] == group_id, 'nobody gid user=nobody' 731 732 self.load('user_group', user='nobody', group=group) 733 734 obj = self.getjson()['body'] 735 assert obj['UID'] == nobody_uid, ( 736 'nobody uid user=nobody group=%s' % group 737 ) 738 739 assert obj['GID'] == group_id, ( 740 'nobody gid user=nobody group=%s' % group 741 ) 742 743 self.load('user_group', group=group) 744 745 obj = self.getjson()['body'] 746 assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group 747 748 assert obj['GID'] == group_id, 'nobody gid group=%s' % group 749 750 self.load('user_group', user='root') 751 752 obj = self.getjson()['body'] 753 assert obj['UID'] == 0, 'root uid user=root' 754 assert obj['GID'] == 0, 'root gid user=root' 755 756 group = 'root' 757 758 try: 759 grp.getgrnam(group) 760 group = True 761 except KeyError: 762 group = False 763 764 if group: 765 self.load('user_group', user='root', group='root') 766 767 obj = self.getjson()['body'] 768 assert obj['UID'] == 0, 'root uid user=root group=root' 769 assert obj['GID'] == 0, 'root gid user=root group=root' 770 771 self.load('user_group', group='root') 772 773 obj = self.getjson()['body'] 774 assert obj['UID'] == nobody_uid, 'root uid group=root' 775 assert obj['GID'] == 0, 'root gid group=root' 776 777 def test_python_application_callable(self, skip_alert): 778 skip_alert(r'Python failed to get "blah" from module') 779 self.load('callable') 780 781 assert self.get()['status'] == 204, 'default application response' 782 783 self.load('callable', callable="app") 784 785 assert self.get()['status'] == 200, 'callable response' 786 787 self.load('callable', callable="blah") 788 789 assert self.get()['status'] not in [200, 204], 'callable response inv' 790 791 def test_python_application_path(self): 792 self.load('path') 793 794 def set_path(path): 795 assert 'success' in self.conf(path, 'applications/path/path') 796 797 def get_path(): 798 return self.get()['body'].split(os.pathsep) 799 800 default_path = self.conf_get('/config/applications/path/path') 801 assert 'success' in self.conf( 802 {"PYTHONPATH": default_path}, 803 '/config/applications/path/environment', 804 ) 805 806 self.conf_delete('/config/applications/path/path') 807 sys_path = get_path() 808 809 set_path('"/blah"') 810 assert ['/blah', *sys_path] == get_path(), 'check path' 811 812 set_path('"/new"') 813 assert ['/new', *sys_path] == get_path(), 'check path update' 814 815 set_path('["/blah1", "/blah2"]') 816 assert [ 817 '/blah1', 818 '/blah2', 819 *sys_path, 820 ] == get_path(), 'check path array' 821 822 def test_python_application_path_invalid(self): 823 self.load('path') 824 825 def check_path(path): 826 assert 'error' in self.conf(path, 'applications/path/path') 827 828 check_path('{}') 829 check_path('["/blah", []]') 830 831 def test_python_application_threads(self): 832 self.load('threads', threads=4) 833 834 socks = [] 835 836 for i in range(4): 837 (_, sock) = self.get( 838 headers={ 839 'Host': 'localhost', 840 'X-Delay': '2', 841 'Connection': 'close', 842 }, 843 no_recv=True, 844 start=True, 845 ) 846 847 socks.append(sock) 848 849 threads = set() 850 851 for sock in socks: 852 resp = self.recvall(sock).decode('utf-8') 853 854 self.log_in(resp) 855 856 resp = self._resp_to_dict(resp) 857 858 assert resp['status'] == 200, 'status' 859 860 threads.add(resp['headers']['X-Thread']) 861 862 assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' 863 864 sock.close() 865 866 assert len(socks) == len(threads), 'threads differs' 867