1import grp 2import os 3import pwd 4import re 5import time 6 7import pytest 8from unit.applications.lang.python import TestApplicationPython 9 10 11class TestPythonApplication(TestApplicationPython): 12 prerequisites = {'modules': {'python': 'all'}} 13 14 def test_python_application_variables(self): 15 self.load('variables') 16 17 body = 'Test body string.' 18 19 resp = self.http( 20 b"""POST / HTTP/1.1 21Host: localhost 22Content-Length: %d 23Custom-Header: blah 24Custom-hEader: Blah 25Content-Type: text/html 26Connection: close 27custom-header: BLAH 28 29%s""" 30 % (len(body), body.encode()), 31 raw=True, 32 ) 33 34 assert resp['status'] == 200, 'status' 35 headers = resp['headers'] 36 header_server = headers.pop('Server') 37 assert re.search(r'Unit/[\d\.]+', header_server), 'server header' 38 assert ( 39 headers.pop('Server-Software') == header_server 40 ), 'server software header' 41 42 date = headers.pop('Date') 43 assert date[-4:] == ' GMT', 'date header timezone' 44 assert ( 45 abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5 46 ), 'date header' 47 48 assert headers == { 49 'Connection': 'close', 50 'Content-Length': str(len(body)), 51 'Content-Type': 'text/html', 52 'Request-Method': 'POST', 53 'Request-Uri': '/', 54 'Http-Host': 'localhost', 55 'Server-Protocol': 'HTTP/1.1', 56 'Custom-Header': 'blah, Blah, BLAH', 57 'Wsgi-Version': '(1, 0)', 58 'Wsgi-Url-Scheme': 'http', 59 'Wsgi-Multithread': 'False', 60 'Wsgi-Multiprocess': 'True', 61 'Wsgi-Run-Once': 'False', 62 }, 'headers' 63 assert resp['body'] == body, 'body' 64 65 def test_python_application_query_string(self): 66 self.load('query_string') 67 68 resp = self.get(url='/?var1=val1&var2=val2') 69 70 assert ( 71 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 72 ), 'Query-String header' 73 74 def test_python_application_query_string_space(self): 75 self.load('query_string') 76 77 resp = self.get(url='/ ?var1=val1&var2=val2') 78 assert ( 79 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 80 ), 'Query-String space' 81 82 resp = self.get(url='/ %20?var1=val1&var2=val2') 83 assert ( 84 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 85 ), 'Query-String space 2' 86 87 resp = self.get(url='/ %20 ?var1=val1&var2=val2') 88 assert ( 89 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 90 ), 'Query-String space 3' 91 92 resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2') 93 assert ( 94 resp['headers']['Query-String'] == ' var1= val1 & var2=val2' 95 ), 'Query-String space 4' 96 97 def test_python_application_query_string_empty(self): 98 self.load('query_string') 99 100 resp = self.get(url='/?') 101 102 assert resp['status'] == 200, 'query string empty status' 103 assert resp['headers']['Query-String'] == '', 'query string empty' 104 105 def test_python_application_query_string_absent(self): 106 self.load('query_string') 107 108 resp = self.get() 109 110 assert resp['status'] == 200, 'query string absent status' 111 assert resp['headers']['Query-String'] == '', 'query string absent' 112 113 @pytest.mark.skip('not yet') 114 def test_python_application_server_port(self): 115 self.load('server_port') 116 117 assert ( 118 self.get()['headers']['Server-Port'] == '7080' 119 ), 'Server-Port header' 120 121 @pytest.mark.skip('not yet') 122 def test_python_application_working_directory_invalid(self): 123 self.load('empty') 124 125 assert 'success' in self.conf( 126 '"/blah"', 'applications/empty/working_directory' 127 ), 'configure invalid working_directory' 128 129 assert self.get()['status'] == 500, 'status' 130 131 def test_python_application_204_transfer_encoding(self): 132 self.load('204_no_content') 133 134 assert ( 135 'Transfer-Encoding' not in self.get()['headers'] 136 ), '204 header transfer encoding' 137 138 def test_python_application_ctx_iter_atexit(self): 139 self.load('ctx_iter_atexit') 140 141 resp = self.post( 142 headers={ 143 'Host': 'localhost', 144 'Connection': 'close', 145 'Content-Type': 'text/html', 146 }, 147 body='0123456789', 148 ) 149 150 assert resp['status'] == 200, 'ctx iter status' 151 assert resp['body'] == '0123456789', 'ctx iter body' 152 153 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 154 155 assert ( 156 self.wait_for_record(r'RuntimeError') is not None 157 ), 'ctx iter atexit' 158 159 def test_python_keepalive_body(self): 160 self.load('mirror') 161 162 assert self.get()['status'] == 200, 'init' 163 164 body = '0123456789' * 500 165 (resp, sock) = self.post( 166 headers={ 167 'Host': 'localhost', 168 'Connection': 'keep-alive', 169 'Content-Type': 'text/html', 170 }, 171 start=True, 172 body=body, 173 read_timeout=1, 174 ) 175 176 assert resp['body'] == body, 'keep-alive 1' 177 178 body = '0123456789' 179 resp = self.post( 180 headers={ 181 'Host': 'localhost', 182 'Connection': 'close', 183 'Content-Type': 'text/html', 184 }, 185 sock=sock, 186 body=body, 187 ) 188 189 assert resp['body'] == body, 'keep-alive 2' 190 191 def test_python_keepalive_reconfigure(self): 192 self.load('mirror') 193 194 assert self.get()['status'] == 200, 'init' 195 196 body = '0123456789' 197 conns = 3 198 socks = [] 199 200 for i in range(conns): 201 (resp, sock) = self.post( 202 headers={ 203 'Host': 'localhost', 204 'Connection': 'keep-alive', 205 'Content-Type': 'text/html', 206 }, 207 start=True, 208 body=body, 209 read_timeout=1, 210 ) 211 212 assert resp['body'] == body, 'keep-alive open' 213 214 self.load('mirror', processes=i + 1) 215 216 socks.append(sock) 217 218 for i in range(conns): 219 (resp, sock) = self.post( 220 headers={ 221 'Host': 'localhost', 222 'Connection': 'keep-alive', 223 'Content-Type': 'text/html', 224 }, 225 start=True, 226 sock=socks[i], 227 body=body, 228 read_timeout=1, 229 ) 230 231 assert resp['body'] == body, 'keep-alive request' 232 233 self.load('mirror', processes=i + 1) 234 235 for i in range(conns): 236 resp = self.post( 237 headers={ 238 'Host': 'localhost', 239 'Connection': 'close', 240 'Content-Type': 'text/html', 241 }, 242 sock=socks[i], 243 body=body, 244 ) 245 246 assert resp['body'] == body, 'keep-alive close' 247 248 self.load('mirror', processes=i + 1) 249 250 def test_python_keepalive_reconfigure_2(self): 251 self.load('mirror') 252 253 assert self.get()['status'] == 200, 'init' 254 255 body = '0123456789' 256 257 (resp, sock) = self.post( 258 headers={ 259 'Host': 'localhost', 260 'Connection': 'keep-alive', 261 'Content-Type': 'text/html', 262 }, 263 start=True, 264 body=body, 265 read_timeout=1, 266 ) 267 268 assert resp['body'] == body, 'reconfigure 2 keep-alive 1' 269 270 self.load('empty') 271 272 assert self.get()['status'] == 200, 'init' 273 274 (resp, sock) = self.post( 275 headers={ 276 'Host': 'localhost', 277 'Connection': 'close', 278 'Content-Type': 'text/html', 279 }, 280 start=True, 281 sock=sock, 282 body=body, 283 ) 284 285 assert resp['status'] == 200, 'reconfigure 2 keep-alive 2' 286 assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body' 287 288 assert 'success' in self.conf( 289 {"listeners": {}, "applications": {}} 290 ), 'reconfigure 2 clear configuration' 291 292 resp = self.get(sock=sock) 293 294 assert resp == {}, 'reconfigure 2 keep-alive 3' 295 296 def test_python_keepalive_reconfigure_3(self): 297 self.load('empty') 298 299 assert self.get()['status'] == 200, 'init' 300 301 (_, sock) = self.http( 302 b"""GET / HTTP/1.1 303""", 304 start=True, 305 raw=True, 306 no_recv=True, 307 ) 308 309 assert self.get()['status'] == 200 310 311 assert 'success' in self.conf( 312 {"listeners": {}, "applications": {}} 313 ), 'reconfigure 3 clear configuration' 314 315 resp = self.http( 316 b"""Host: localhost 317Connection: close 318 319""", 320 sock=sock, 321 raw=True, 322 ) 323 324 assert resp['status'] == 200, 'reconfigure 3' 325 326 def test_python_atexit(self): 327 self.load('atexit') 328 329 self.get() 330 331 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 332 333 assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' 334 335 def test_python_process_switch(self): 336 self.load('delayed', processes=2) 337 338 self.get( 339 headers={ 340 'Host': 'localhost', 341 'Content-Length': '0', 342 'X-Delay': '5', 343 'Connection': 'close', 344 }, 345 no_recv=True, 346 ) 347 348 headers_delay_1 = { 349 'Connection': 'close', 350 'Host': 'localhost', 351 'Content-Length': '0', 352 'X-Delay': '1', 353 } 354 355 self.get(headers=headers_delay_1, no_recv=True) 356 357 time.sleep(0.5) 358 359 for _ in range(10): 360 self.get(headers=headers_delay_1, no_recv=True) 361 362 self.get(headers=headers_delay_1) 363 364 @pytest.mark.skip('not yet') 365 def test_python_application_start_response_exit(self): 366 self.load('start_response_exit') 367 368 assert self.get()['status'] == 500, 'start response exit' 369 370 def test_python_application_input_iter(self): 371 self.load('input_iter') 372 373 body = '''0123456789 374next line 375 376last line''' 377 378 resp = self.post(body=body) 379 assert resp['body'] == body, 'input iter' 380 assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines' 381 382 def test_python_application_input_readline(self): 383 self.load('input_readline') 384 385 body = '''0123456789 386next line 387 388last line''' 389 390 resp = self.post(body=body) 391 assert resp['body'] == body, 'input readline' 392 assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines' 393 394 def test_python_application_input_readline_size(self): 395 self.load('input_readline_size') 396 397 body = '''0123456789 398next line 399 400last line''' 401 402 assert self.post(body=body)['body'] == body, 'input readline size' 403 assert ( 404 self.post(body='0123')['body'] == '0123' 405 ), 'input readline size less' 406 407 def test_python_application_input_readlines(self): 408 self.load('input_readlines') 409 410 body = '''0123456789 411next line 412 413last line''' 414 415 resp = self.post(body=body) 416 assert resp['body'] == body, 'input readlines' 417 assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines' 418 419 def test_python_application_input_readlines_huge(self): 420 self.load('input_readlines') 421 422 body = ( 423 '''0123456789 abcdefghi 424next line: 0123456789 abcdefghi 425 426last line: 987654321 427''' 428 * 512 429 ) 430 431 assert ( 432 self.post(body=body, read_buffer_size=16384)['body'] == body 433 ), 'input readlines huge' 434 435 def test_python_application_input_read_length(self): 436 self.load('input_read_length') 437 438 body = '0123456789' 439 440 resp = self.post( 441 headers={ 442 'Host': 'localhost', 443 'Input-Length': '5', 444 'Connection': 'close', 445 }, 446 body=body, 447 ) 448 449 assert resp['body'] == body[:5], 'input read length lt body' 450 451 resp = self.post( 452 headers={ 453 'Host': 'localhost', 454 'Input-Length': '15', 455 'Connection': 'close', 456 }, 457 body=body, 458 ) 459 460 assert resp['body'] == body, 'input read length gt body' 461 462 resp = self.post( 463 headers={ 464 'Host': 'localhost', 465 'Input-Length': '0', 466 'Connection': 'close', 467 }, 468 body=body, 469 ) 470 471 assert resp['body'] == '', 'input read length zero' 472 473 resp = self.post( 474 headers={ 475 'Host': 'localhost', 476 'Input-Length': '-1', 477 'Connection': 'close', 478 }, 479 body=body, 480 ) 481 482 assert resp['body'] == body, 'input read length negative' 483 484 @pytest.mark.skip('not yet') 485 def test_python_application_errors_write(self): 486 self.load('errors_write') 487 488 self.get() 489 490 assert ( 491 self.wait_for_record(r'\[error\].+Error in application\.') 492 is not None 493 ), 'errors write' 494 495 def test_python_application_body_array(self): 496 self.load('body_array') 497 498 assert self.get()['body'] == '0123456789', 'body array' 499 500 def test_python_application_body_io(self): 501 self.load('body_io') 502 503 assert self.get()['body'] == '0123456789', 'body io' 504 505 def test_python_application_body_io_file(self): 506 self.load('body_io_file') 507 508 assert self.get()['body'] == 'body\n', 'body io file' 509 510 @pytest.mark.skip('not yet') 511 def test_python_application_syntax_error(self, skip_alert): 512 skip_alert(r'Python failed to import module "wsgi"') 513 self.load('syntax_error') 514 515 assert self.get()['status'] == 500, 'syntax error' 516 517 def test_python_application_loading_error(self, skip_alert): 518 skip_alert(r'Python failed to import module "blah"') 519 520 self.load('empty', module="blah") 521 522 assert self.get()['status'] == 503, 'loading error' 523 524 def test_python_application_close(self): 525 self.load('close') 526 527 self.get() 528 529 assert self.wait_for_record(r'Close called\.') is not None, 'close' 530 531 def test_python_application_close_error(self): 532 self.load('close_error') 533 534 self.get() 535 536 assert ( 537 self.wait_for_record(r'Close called\.') is not None 538 ), 'close error' 539 540 def test_python_application_not_iterable(self): 541 self.load('not_iterable') 542 543 self.get() 544 545 assert ( 546 self.wait_for_record( 547 r'\[error\].+the application returned not an iterable object' 548 ) 549 is not None 550 ), 'not iterable' 551 552 def test_python_application_write(self): 553 self.load('write') 554 555 assert self.get()['body'] == '0123456789', 'write' 556 557 def test_python_application_threading(self): 558 """wait_for_record() timeouts after 5s while every thread works at 559 least 3s. So without releasing GIL test should fail. 560 """ 561 562 self.load('threading') 563 564 for _ in range(10): 565 self.get(no_recv=True) 566 567 assert ( 568 self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None 569 ), 'last thread finished' 570 571 def test_python_application_iter_exception(self): 572 self.load('iter_exception') 573 574 # Default request doesn't lead to the exception. 575 576 resp = self.get( 577 headers={ 578 'Host': 'localhost', 579 'X-Skip': '9', 580 'X-Chunked': '1', 581 'Connection': 'close', 582 } 583 ) 584 assert resp['status'] == 200, 'status' 585 assert resp['body'] == 'XXXXXXX', 'body' 586 587 # Exception before start_response(). 588 589 assert self.get()['status'] == 503, 'error' 590 591 assert self.wait_for_record(r'Traceback') is not None, 'traceback' 592 assert ( 593 self.wait_for_record(r'raise Exception\(\'first exception\'\)') 594 is not None 595 ), 'first exception raise' 596 assert len(self.findall(r'Traceback')) == 1, 'traceback count 1' 597 598 # Exception after start_response(), before first write(). 599 600 assert ( 601 self.get( 602 headers={ 603 'Host': 'localhost', 604 'X-Skip': '1', 605 'Connection': 'close', 606 } 607 )['status'] 608 == 503 609 ), 'error 2' 610 611 assert ( 612 self.wait_for_record(r'raise Exception\(\'second exception\'\)') 613 is not None 614 ), 'exception raise second' 615 assert len(self.findall(r'Traceback')) == 2, 'traceback count 2' 616 617 # Exception after first write(), before first __next__(). 618 619 _, sock = self.get( 620 headers={ 621 'Host': 'localhost', 622 'X-Skip': '2', 623 'Connection': 'keep-alive', 624 }, 625 start=True, 626 ) 627 628 assert ( 629 self.wait_for_record(r'raise Exception\(\'third exception\'\)') 630 is not None 631 ), 'exception raise third' 632 assert len(self.findall(r'Traceback')) == 3, 'traceback count 3' 633 634 assert self.get(sock=sock) == {}, 'closed connection' 635 636 # Exception after first write(), before first __next__(), 637 # chunked (incomplete body). 638 639 resp = self.get( 640 headers={ 641 'Host': 'localhost', 642 'X-Skip': '2', 643 'X-Chunked': '1', 644 'Connection': 'close', 645 }, 646 raw_resp=True, 647 ) 648 if resp: 649 assert resp[-5:] != '0\r\n\r\n', 'incomplete body' 650 assert len(self.findall(r'Traceback')) == 4, 'traceback count 4' 651 652 # Exception in __next__(). 653 654 _, sock = self.get( 655 headers={ 656 'Host': 'localhost', 657 'X-Skip': '3', 658 'Connection': 'keep-alive', 659 }, 660 start=True, 661 ) 662 663 assert ( 664 self.wait_for_record(r'raise Exception\(\'next exception\'\)') 665 is not None 666 ), 'exception raise next' 667 assert len(self.findall(r'Traceback')) == 5, 'traceback count 5' 668 669 assert self.get(sock=sock) == {}, 'closed connection 2' 670 671 # Exception in __next__(), chunked (incomplete body). 672 673 resp = self.get( 674 headers={ 675 'Host': 'localhost', 676 'X-Skip': '3', 677 'X-Chunked': '1', 678 'Connection': 'close', 679 }, 680 raw_resp=True, 681 ) 682 if resp: 683 assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2' 684 assert len(self.findall(r'Traceback')) == 6, 'traceback count 6' 685 686 # Exception before start_response() and in close(). 687 688 assert ( 689 self.get( 690 headers={ 691 'Host': 'localhost', 692 'X-Not-Skip-Close': '1', 693 'Connection': 'close', 694 } 695 )['status'] 696 == 503 697 ), 'error' 698 699 assert ( 700 self.wait_for_record(r'raise Exception\(\'close exception\'\)') 701 is not None 702 ), 'exception raise close' 703 assert len(self.findall(r'Traceback')) == 8, 'traceback count 8' 704 705 def test_python_user_group(self, is_su): 706 if not is_su: 707 pytest.skip('requires root') 708 709 nobody_uid = pwd.getpwnam('nobody').pw_uid 710 711 group = 'nobody' 712 713 try: 714 group_id = grp.getgrnam(group).gr_gid 715 except KeyError: 716 group = 'nogroup' 717 group_id = grp.getgrnam(group).gr_gid 718 719 self.load('user_group') 720 721 obj = self.getjson()['body'] 722 assert obj['UID'] == nobody_uid, 'nobody uid' 723 assert obj['GID'] == group_id, 'nobody gid' 724 725 self.load('user_group', user='nobody') 726 727 obj = self.getjson()['body'] 728 assert obj['UID'] == nobody_uid, 'nobody uid user=nobody' 729 assert obj['GID'] == group_id, 'nobody gid user=nobody' 730 731 self.load('user_group', user='nobody', group=group) 732 733 obj = self.getjson()['body'] 734 assert obj['UID'] == nobody_uid, ( 735 'nobody uid user=nobody group=%s' % group 736 ) 737 738 assert obj['GID'] == group_id, 'nobody gid user=nobody group=%s' % group 739 740 self.load('user_group', group=group) 741 742 obj = self.getjson()['body'] 743 assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group 744 745 assert obj['GID'] == group_id, 'nobody gid group=%s' % group 746 747 self.load('user_group', user='root') 748 749 obj = self.getjson()['body'] 750 assert obj['UID'] == 0, 'root uid user=root' 751 assert obj['GID'] == 0, 'root gid user=root' 752 753 group = 'root' 754 755 try: 756 grp.getgrnam(group) 757 group = True 758 except KeyError: 759 group = False 760 761 if group: 762 self.load('user_group', user='root', group='root') 763 764 obj = self.getjson()['body'] 765 assert obj['UID'] == 0, 'root uid user=root group=root' 766 assert obj['GID'] == 0, 'root gid user=root group=root' 767 768 self.load('user_group', group='root') 769 770 obj = self.getjson()['body'] 771 assert obj['UID'] == nobody_uid, 'root uid group=root' 772 assert obj['GID'] == 0, 'root gid group=root' 773 774 def test_python_application_callable(self, skip_alert): 775 skip_alert(r'Python failed to get "blah" from module') 776 self.load('callable') 777 778 assert self.get()['status'] == 204, 'default application response' 779 780 self.load('callable', callable="app") 781 782 assert self.get()['status'] == 200, 'callable response' 783 784 self.load('callable', callable="blah") 785 786 assert self.get()['status'] not in [200, 204], 'callable response inv' 787 788 def test_python_application_path(self): 789 self.load('path') 790 791 def set_path(path): 792 assert 'success' in self.conf(path, 'applications/path/path') 793 794 def get_path(): 795 return self.get()['body'].split(os.pathsep) 796 797 default_path = self.conf_get('/config/applications/path/path') 798 assert 'success' in self.conf( 799 {"PYTHONPATH": default_path}, 800 '/config/applications/path/environment', 801 ) 802 803 self.conf_delete('/config/applications/path/path') 804 sys_path = get_path() 805 806 set_path('"/blah"') 807 assert ['/blah', *sys_path] == get_path(), 'check path' 808 809 set_path('"/new"') 810 assert ['/new', *sys_path] == get_path(), 'check path update' 811 812 set_path('["/blah1", "/blah2"]') 813 assert [ 814 '/blah1', 815 '/blah2', 816 *sys_path, 817 ] == get_path(), 'check path array' 818 819 def test_python_application_path_invalid(self): 820 self.load('path') 821 822 def check_path(path): 823 assert 'error' in self.conf(path, 'applications/path/path') 824 825 check_path('{}') 826 check_path('["/blah", []]') 827 828 def test_python_application_threads(self): 829 self.load('threads', threads=4) 830 831 socks = [] 832 833 for i in range(4): 834 (_, sock) = self.get( 835 headers={ 836 'Host': 'localhost', 837 'X-Delay': '2', 838 'Connection': 'close', 839 }, 840 no_recv=True, 841 start=True, 842 ) 843 844 socks.append(sock) 845 846 threads = set() 847 848 for sock in socks: 849 resp = self.recvall(sock).decode('utf-8') 850 851 self.log_in(resp) 852 853 resp = self._resp_to_dict(resp) 854 855 assert resp['status'] == 200, 'status' 856 857 threads.add(resp['headers']['X-Thread']) 858 859 assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' 860 861 sock.close() 862 863 assert len(socks) == len(threads), 'threads differs' 864