1import grp 2import os 3import pwd 4import re 5import time 6 7import pytest 8 9from unit.applications.lang.python import TestApplicationPython 10from unit.option import option 11 12 13class TestPythonApplication(TestApplicationPython): 14 prerequisites = {'modules': {'python': 'all'}} 15 16 def findall(self, pattern): 17 with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: 18 return re.findall(pattern, f.read()) 19 20 def test_python_application_variables(self): 21 self.load('variables') 22 23 body = 'Test body string.' 24 25 resp = self.http( 26 b"""POST / HTTP/1.1 27Host: localhost 28Content-Length: %d 29Custom-Header: blah 30Custom-hEader: Blah 31Content-Type: text/html 32Connection: close 33custom-header: BLAH 34 35%s""" 36 % (len(body), body.encode()), 37 raw=True, 38 ) 39 40 assert resp['status'] == 200, 'status' 41 headers = resp['headers'] 42 header_server = headers.pop('Server') 43 assert re.search(r'Unit/[\d\.]+', header_server), 'server header' 44 assert ( 45 headers.pop('Server-Software') == header_server 46 ), 'server software header' 47 48 date = headers.pop('Date') 49 assert date[-4:] == ' GMT', 'date header timezone' 50 assert ( 51 abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5 52 ), 'date header' 53 54 assert headers == { 55 'Connection': 'close', 56 'Content-Length': str(len(body)), 57 'Content-Type': 'text/html', 58 'Request-Method': 'POST', 59 'Request-Uri': '/', 60 'Http-Host': 'localhost', 61 'Server-Protocol': 'HTTP/1.1', 62 'Custom-Header': 'blah, Blah, BLAH', 63 'Wsgi-Version': '(1, 0)', 64 'Wsgi-Url-Scheme': 'http', 65 'Wsgi-Multithread': 'False', 66 'Wsgi-Multiprocess': 'True', 67 'Wsgi-Run-Once': 'False', 68 }, 'headers' 69 assert resp['body'] == body, 'body' 70 71 def test_python_application_query_string(self): 72 self.load('query_string') 73 74 resp = self.get(url='/?var1=val1&var2=val2') 75 76 assert ( 77 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 78 ), 'Query-String header' 79 80 def test_python_application_query_string_space(self): 81 self.load('query_string') 82 83 resp = self.get(url='/ ?var1=val1&var2=val2') 84 assert ( 85 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 86 ), 'Query-String space' 87 88 resp = self.get(url='/ %20?var1=val1&var2=val2') 89 assert ( 90 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 91 ), 'Query-String space 2' 92 93 resp = self.get(url='/ %20 ?var1=val1&var2=val2') 94 assert ( 95 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 96 ), 'Query-String space 3' 97 98 resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2') 99 assert ( 100 resp['headers']['Query-String'] == ' var1= val1 & var2=val2' 101 ), 'Query-String space 4' 102 103 def test_python_application_query_string_empty(self): 104 self.load('query_string') 105 106 resp = self.get(url='/?') 107 108 assert resp['status'] == 200, 'query string empty status' 109 assert resp['headers']['Query-String'] == '', 'query string empty' 110 111 def test_python_application_query_string_absent(self): 112 self.load('query_string') 113 114 resp = self.get() 115 116 assert resp['status'] == 200, 'query string absent status' 117 assert resp['headers']['Query-String'] == '', 'query string absent' 118 119 @pytest.mark.skip('not yet') 120 def test_python_application_server_port(self): 121 self.load('server_port') 122 123 assert ( 124 self.get()['headers']['Server-Port'] == '7080' 125 ), 'Server-Port header' 126 127 @pytest.mark.skip('not yet') 128 def test_python_application_working_directory_invalid(self): 129 self.load('empty') 130 131 assert 'success' in self.conf( 132 '"/blah"', 'applications/empty/working_directory' 133 ), 'configure invalid working_directory' 134 135 assert self.get()['status'] == 500, 'status' 136 137 def test_python_application_204_transfer_encoding(self): 138 self.load('204_no_content') 139 140 assert ( 141 'Transfer-Encoding' not in self.get()['headers'] 142 ), '204 header transfer encoding' 143 144 def test_python_application_ctx_iter_atexit(self): 145 self.load('ctx_iter_atexit') 146 147 resp = self.post( 148 headers={ 149 'Host': 'localhost', 150 'Connection': 'close', 151 'Content-Type': 'text/html', 152 }, 153 body='0123456789', 154 ) 155 156 assert resp['status'] == 200, 'ctx iter status' 157 assert resp['body'] == '0123456789', 'ctx iter body' 158 159 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 160 161 assert ( 162 self.wait_for_record(r'RuntimeError') is not None 163 ), 'ctx iter atexit' 164 165 def test_python_keepalive_body(self): 166 self.load('mirror') 167 168 assert self.get()['status'] == 200, 'init' 169 170 body = '0123456789' * 500 171 (resp, sock) = self.post( 172 headers={ 173 'Host': 'localhost', 174 'Connection': 'keep-alive', 175 'Content-Type': 'text/html', 176 }, 177 start=True, 178 body=body, 179 read_timeout=1, 180 ) 181 182 assert resp['body'] == body, 'keep-alive 1' 183 184 body = '0123456789' 185 resp = self.post( 186 headers={ 187 'Host': 'localhost', 188 'Connection': 'close', 189 'Content-Type': 'text/html', 190 }, 191 sock=sock, 192 body=body, 193 ) 194 195 assert resp['body'] == body, 'keep-alive 2' 196 197 def test_python_keepalive_reconfigure(self): 198 self.load('mirror') 199 200 assert self.get()['status'] == 200, 'init' 201 202 body = '0123456789' 203 conns = 3 204 socks = [] 205 206 for i in range(conns): 207 (resp, sock) = self.post( 208 headers={ 209 'Host': 'localhost', 210 'Connection': 'keep-alive', 211 'Content-Type': 'text/html', 212 }, 213 start=True, 214 body=body, 215 read_timeout=1, 216 ) 217 218 assert resp['body'] == body, 'keep-alive open' 219 220 self.load('mirror', processes=i + 1) 221 222 socks.append(sock) 223 224 for i in range(conns): 225 (resp, sock) = self.post( 226 headers={ 227 'Host': 'localhost', 228 'Connection': 'keep-alive', 229 'Content-Type': 'text/html', 230 }, 231 start=True, 232 sock=socks[i], 233 body=body, 234 read_timeout=1, 235 ) 236 237 assert resp['body'] == body, 'keep-alive request' 238 239 self.load('mirror', processes=i + 1) 240 241 for i in range(conns): 242 resp = self.post( 243 headers={ 244 'Host': 'localhost', 245 'Connection': 'close', 246 'Content-Type': 'text/html', 247 }, 248 sock=socks[i], 249 body=body, 250 ) 251 252 assert resp['body'] == body, 'keep-alive close' 253 254 self.load('mirror', processes=i + 1) 255 256 def test_python_keepalive_reconfigure_2(self): 257 self.load('mirror') 258 259 assert self.get()['status'] == 200, 'init' 260 261 body = '0123456789' 262 263 (resp, sock) = self.post( 264 headers={ 265 'Host': 'localhost', 266 'Connection': 'keep-alive', 267 'Content-Type': 'text/html', 268 }, 269 start=True, 270 body=body, 271 read_timeout=1, 272 ) 273 274 assert resp['body'] == body, 'reconfigure 2 keep-alive 1' 275 276 self.load('empty') 277 278 assert self.get()['status'] == 200, 'init' 279 280 (resp, sock) = self.post( 281 headers={ 282 'Host': 'localhost', 283 'Connection': 'close', 284 'Content-Type': 'text/html', 285 }, 286 start=True, 287 sock=sock, 288 body=body, 289 ) 290 291 assert resp['status'] == 200, 'reconfigure 2 keep-alive 2' 292 assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body' 293 294 assert 'success' in self.conf( 295 {"listeners": {}, "applications": {}} 296 ), 'reconfigure 2 clear configuration' 297 298 resp = self.get(sock=sock) 299 300 assert resp == {}, 'reconfigure 2 keep-alive 3' 301 302 def test_python_keepalive_reconfigure_3(self): 303 self.load('empty') 304 305 assert self.get()['status'] == 200, 'init' 306 307 (_, sock) = self.http( 308 b"""GET / HTTP/1.1 309""", 310 start=True, 311 raw=True, 312 no_recv=True, 313 ) 314 315 assert self.get()['status'] == 200 316 317 assert 'success' in self.conf( 318 {"listeners": {}, "applications": {}} 319 ), 'reconfigure 3 clear configuration' 320 321 resp = self.http( 322 b"""Host: localhost 323Connection: close 324 325""", 326 sock=sock, 327 raw=True, 328 ) 329 330 assert resp['status'] == 200, 'reconfigure 3' 331 332 def test_python_atexit(self): 333 self.load('atexit') 334 335 self.get() 336 337 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 338 339 assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' 340 341 def test_python_process_switch(self): 342 self.load('delayed', processes=2) 343 344 self.get( 345 headers={ 346 'Host': 'localhost', 347 'Content-Length': '0', 348 'X-Delay': '5', 349 'Connection': 'close', 350 }, 351 no_recv=True, 352 ) 353 354 headers_delay_1 = { 355 'Connection': 'close', 356 'Host': 'localhost', 357 'Content-Length': '0', 358 'X-Delay': '1', 359 } 360 361 self.get(headers=headers_delay_1, no_recv=True) 362 363 time.sleep(0.5) 364 365 for _ in range(10): 366 self.get(headers=headers_delay_1, no_recv=True) 367 368 self.get(headers=headers_delay_1) 369 370 @pytest.mark.skip('not yet') 371 def test_python_application_start_response_exit(self): 372 self.load('start_response_exit') 373 374 assert self.get()['status'] == 500, 'start response exit' 375 376 def test_python_application_input_iter(self): 377 self.load('input_iter') 378 379 body = '''0123456789 380next line 381 382last line''' 383 384 resp = self.post(body=body) 385 assert resp['body'] == body, 'input iter' 386 assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines' 387 388 def test_python_application_input_readline(self): 389 self.load('input_readline') 390 391 body = '''0123456789 392next line 393 394last line''' 395 396 resp = self.post(body=body) 397 assert resp['body'] == body, 'input readline' 398 assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines' 399 400 def test_python_application_input_readline_size(self): 401 self.load('input_readline_size') 402 403 body = '''0123456789 404next line 405 406last line''' 407 408 assert self.post(body=body)['body'] == body, 'input readline size' 409 assert ( 410 self.post(body='0123')['body'] == '0123' 411 ), 'input readline size less' 412 413 def test_python_application_input_readlines(self): 414 self.load('input_readlines') 415 416 body = '''0123456789 417next line 418 419last line''' 420 421 resp = self.post(body=body) 422 assert resp['body'] == body, 'input readlines' 423 assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines' 424 425 def test_python_application_input_readlines_huge(self): 426 self.load('input_readlines') 427 428 body = ( 429 '''0123456789 abcdefghi 430next line: 0123456789 abcdefghi 431 432last line: 987654321 433''' 434 * 512 435 ) 436 437 assert ( 438 self.post(body=body, read_buffer_size=16384)['body'] == body 439 ), 'input readlines huge' 440 441 def test_python_application_input_read_length(self): 442 self.load('input_read_length') 443 444 body = '0123456789' 445 446 resp = self.post( 447 headers={ 448 'Host': 'localhost', 449 'Input-Length': '5', 450 'Connection': 'close', 451 }, 452 body=body, 453 ) 454 455 assert resp['body'] == body[:5], 'input read length lt body' 456 457 resp = self.post( 458 headers={ 459 'Host': 'localhost', 460 'Input-Length': '15', 461 'Connection': 'close', 462 }, 463 body=body, 464 ) 465 466 assert resp['body'] == body, 'input read length gt body' 467 468 resp = self.post( 469 headers={ 470 'Host': 'localhost', 471 'Input-Length': '0', 472 'Connection': 'close', 473 }, 474 body=body, 475 ) 476 477 assert resp['body'] == '', 'input read length zero' 478 479 resp = self.post( 480 headers={ 481 'Host': 'localhost', 482 'Input-Length': '-1', 483 'Connection': 'close', 484 }, 485 body=body, 486 ) 487 488 assert resp['body'] == body, 'input read length negative' 489 490 @pytest.mark.skip('not yet') 491 def test_python_application_errors_write(self): 492 self.load('errors_write') 493 494 self.get() 495 496 assert ( 497 self.wait_for_record(r'\[error\].+Error in application\.') 498 is not None 499 ), 'errors write' 500 501 def test_python_application_body_array(self): 502 self.load('body_array') 503 504 assert self.get()['body'] == '0123456789', 'body array' 505 506 def test_python_application_body_io(self): 507 self.load('body_io') 508 509 assert self.get()['body'] == '0123456789', 'body io' 510 511 def test_python_application_body_io_file(self): 512 self.load('body_io_file') 513 514 assert self.get()['body'] == 'body\n', 'body io file' 515 516 @pytest.mark.skip('not yet') 517 def test_python_application_syntax_error(self, skip_alert): 518 skip_alert(r'Python failed to import module "wsgi"') 519 self.load('syntax_error') 520 521 assert self.get()['status'] == 500, 'syntax error' 522 523 def test_python_application_loading_error(self, skip_alert): 524 skip_alert(r'Python failed to import module "blah"') 525 526 self.load('empty', module="blah") 527 528 assert self.get()['status'] == 503, 'loading error' 529 530 def test_python_application_close(self): 531 self.load('close') 532 533 self.get() 534 535 assert self.wait_for_record(r'Close called\.') is not None, 'close' 536 537 def test_python_application_close_error(self): 538 self.load('close_error') 539 540 self.get() 541 542 assert ( 543 self.wait_for_record(r'Close called\.') is not None 544 ), 'close error' 545 546 def test_python_application_not_iterable(self): 547 self.load('not_iterable') 548 549 self.get() 550 551 assert ( 552 self.wait_for_record( 553 r'\[error\].+the application returned not an iterable object' 554 ) 555 is not None 556 ), 'not iterable' 557 558 def test_python_application_write(self): 559 self.load('write') 560 561 assert self.get()['body'] == '0123456789', 'write' 562 563 def test_python_application_threading(self): 564 """wait_for_record() timeouts after 5s while every thread works at 565 least 3s. So without releasing GIL test should fail. 566 """ 567 568 self.load('threading') 569 570 for _ in range(10): 571 self.get(no_recv=True) 572 573 assert ( 574 self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None 575 ), 'last thread finished' 576 577 def test_python_application_iter_exception(self): 578 self.load('iter_exception') 579 580 # Default request doesn't lead to the exception. 581 582 resp = self.get( 583 headers={ 584 'Host': 'localhost', 585 'X-Skip': '9', 586 'X-Chunked': '1', 587 'Connection': 'close', 588 } 589 ) 590 assert resp['status'] == 200, 'status' 591 assert resp['body'] == 'XXXXXXX', 'body' 592 593 # Exception before start_response(). 594 595 assert self.get()['status'] == 503, 'error' 596 597 assert self.wait_for_record(r'Traceback') is not None, 'traceback' 598 assert ( 599 self.wait_for_record(r'raise Exception\(\'first exception\'\)') 600 is not None 601 ), 'first exception raise' 602 assert len(self.findall(r'Traceback')) == 1, 'traceback count 1' 603 604 # Exception after start_response(), before first write(). 605 606 assert ( 607 self.get( 608 headers={ 609 'Host': 'localhost', 610 'X-Skip': '1', 611 'Connection': 'close', 612 } 613 )['status'] 614 == 503 615 ), 'error 2' 616 617 assert ( 618 self.wait_for_record(r'raise Exception\(\'second exception\'\)') 619 is not None 620 ), 'exception raise second' 621 assert len(self.findall(r'Traceback')) == 2, 'traceback count 2' 622 623 # Exception after first write(), before first __next__(). 624 625 _, sock = self.get( 626 headers={ 627 'Host': 'localhost', 628 'X-Skip': '2', 629 'Connection': 'keep-alive', 630 }, 631 start=True, 632 ) 633 634 assert ( 635 self.wait_for_record(r'raise Exception\(\'third exception\'\)') 636 is not None 637 ), 'exception raise third' 638 assert len(self.findall(r'Traceback')) == 3, 'traceback count 3' 639 640 assert self.get(sock=sock) == {}, 'closed connection' 641 642 # Exception after first write(), before first __next__(), 643 # chunked (incomplete body). 644 645 resp = self.get( 646 headers={ 647 'Host': 'localhost', 648 'X-Skip': '2', 649 'X-Chunked': '1', 650 'Connection': 'close', 651 }, 652 raw_resp=True, 653 ) 654 if resp: 655 assert resp[-5:] != '0\r\n\r\n', 'incomplete body' 656 assert len(self.findall(r'Traceback')) == 4, 'traceback count 4' 657 658 # Exception in __next__(). 659 660 _, sock = self.get( 661 headers={ 662 'Host': 'localhost', 663 'X-Skip': '3', 664 'Connection': 'keep-alive', 665 }, 666 start=True, 667 ) 668 669 assert ( 670 self.wait_for_record(r'raise Exception\(\'next exception\'\)') 671 is not None 672 ), 'exception raise next' 673 assert len(self.findall(r'Traceback')) == 5, 'traceback count 5' 674 675 assert self.get(sock=sock) == {}, 'closed connection 2' 676 677 # Exception in __next__(), chunked (incomplete body). 678 679 resp = self.get( 680 headers={ 681 'Host': 'localhost', 682 'X-Skip': '3', 683 'X-Chunked': '1', 684 'Connection': 'close', 685 }, 686 raw_resp=True, 687 ) 688 if resp: 689 assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2' 690 assert len(self.findall(r'Traceback')) == 6, 'traceback count 6' 691 692 # Exception before start_response() and in close(). 693 694 assert ( 695 self.get( 696 headers={ 697 'Host': 'localhost', 698 'X-Not-Skip-Close': '1', 699 'Connection': 'close', 700 } 701 )['status'] 702 == 503 703 ), 'error' 704 705 assert ( 706 self.wait_for_record(r'raise Exception\(\'close exception\'\)') 707 is not None 708 ), 'exception raise close' 709 assert len(self.findall(r'Traceback')) == 8, 'traceback count 8' 710 711 def test_python_user_group(self, is_su): 712 if not is_su: 713 pytest.skip('requires root') 714 715 nobody_uid = pwd.getpwnam('nobody').pw_uid 716 717 group = 'nobody' 718 719 try: 720 group_id = grp.getgrnam(group).gr_gid 721 except KeyError: 722 group = 'nogroup' 723 group_id = grp.getgrnam(group).gr_gid 724 725 self.load('user_group') 726 727 obj = self.getjson()['body'] 728 assert obj['UID'] == nobody_uid, 'nobody uid' 729 assert obj['GID'] == group_id, 'nobody gid' 730 731 self.load('user_group', user='nobody') 732 733 obj = self.getjson()['body'] 734 assert obj['UID'] == nobody_uid, 'nobody uid user=nobody' 735 assert obj['GID'] == group_id, 'nobody gid user=nobody' 736 737 self.load('user_group', user='nobody', group=group) 738 739 obj = self.getjson()['body'] 740 assert obj['UID'] == nobody_uid, ( 741 'nobody uid user=nobody group=%s' % group 742 ) 743 744 assert obj['GID'] == group_id, ( 745 'nobody gid user=nobody group=%s' % group 746 ) 747 748 self.load('user_group', group=group) 749 750 obj = self.getjson()['body'] 751 assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group 752 753 assert obj['GID'] == group_id, 'nobody gid group=%s' % group 754 755 self.load('user_group', user='root') 756 757 obj = self.getjson()['body'] 758 assert obj['UID'] == 0, 'root uid user=root' 759 assert obj['GID'] == 0, 'root gid user=root' 760 761 group = 'root' 762 763 try: 764 grp.getgrnam(group) 765 group = True 766 except KeyError: 767 group = False 768 769 if group: 770 self.load('user_group', user='root', group='root') 771 772 obj = self.getjson()['body'] 773 assert obj['UID'] == 0, 'root uid user=root group=root' 774 assert obj['GID'] == 0, 'root gid user=root group=root' 775 776 self.load('user_group', group='root') 777 778 obj = self.getjson()['body'] 779 assert obj['UID'] == nobody_uid, 'root uid group=root' 780 assert obj['GID'] == 0, 'root gid group=root' 781 782 def test_python_application_callable(self, skip_alert): 783 skip_alert(r'Python failed to get "blah" from module') 784 self.load('callable') 785 786 assert self.get()['status'] == 204, 'default application response' 787 788 self.load('callable', callable="app") 789 790 assert self.get()['status'] == 200, 'callable response' 791 792 self.load('callable', callable="blah") 793 794 assert self.get()['status'] not in [200, 204], 'callable response inv' 795 796 def test_python_application_path(self): 797 self.load('path') 798 799 def set_path(path): 800 assert 'success' in self.conf(path, 'applications/path/path') 801 802 def get_path(): 803 return self.get()['body'].split(os.pathsep) 804 805 default_path = self.conf_get('/config/applications/path/path') 806 assert 'success' in self.conf( 807 {"PYTHONPATH": default_path}, 808 '/config/applications/path/environment', 809 ) 810 811 self.conf_delete('/config/applications/path/path') 812 sys_path = get_path() 813 814 set_path('"/blah"') 815 assert ['/blah', *sys_path] == get_path(), 'check path' 816 817 set_path('"/new"') 818 assert ['/new', *sys_path] == get_path(), 'check path update' 819 820 set_path('["/blah1", "/blah2"]') 821 assert [ 822 '/blah1', 823 '/blah2', 824 *sys_path, 825 ] == get_path(), 'check path array' 826 827 def test_python_application_path_invalid(self): 828 self.load('path') 829 830 def check_path(path): 831 assert 'error' in self.conf(path, 'applications/path/path') 832 833 check_path('{}') 834 check_path('["/blah", []]') 835 836 def test_python_application_threads(self): 837 self.load('threads', threads=4) 838 839 socks = [] 840 841 for i in range(4): 842 (_, sock) = self.get( 843 headers={ 844 'Host': 'localhost', 845 'X-Delay': '2', 846 'Connection': 'close', 847 }, 848 no_recv=True, 849 start=True, 850 ) 851 852 socks.append(sock) 853 854 threads = set() 855 856 for sock in socks: 857 resp = self.recvall(sock).decode('utf-8') 858 859 self.log_in(resp) 860 861 resp = self._resp_to_dict(resp) 862 863 assert resp['status'] == 200, 'status' 864 865 threads.add(resp['headers']['X-Thread']) 866 867 assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' 868 869 sock.close() 870 871 assert len(socks) == len(threads), 'threads differs' 872