1import grp 2import os 3import pwd 4import re 5import time 6 7import pytest 8from unit.applications.lang.python import TestApplicationPython 9from unit.option import option 10 11 12class TestPythonApplication(TestApplicationPython): 13 prerequisites = {'modules': {'python': 'all'}} 14 15 def findall(self, pattern): 16 with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: 17 return re.findall(pattern, f.read()) 18 19 def test_python_application_variables(self): 20 self.load('variables') 21 22 body = 'Test body string.' 23 24 resp = self.http( 25 b"""POST / HTTP/1.1 26Host: localhost 27Content-Length: %d 28Custom-Header: blah 29Custom-hEader: Blah 30Content-Type: text/html 31Connection: close 32custom-header: BLAH 33 34%s""" % (len(body), body.encode()), 35 raw=True, 36 ) 37 38 assert resp['status'] == 200, 'status' 39 headers = resp['headers'] 40 header_server = headers.pop('Server') 41 assert re.search(r'Unit/[\d\.]+', header_server), 'server header' 42 assert ( 43 headers.pop('Server-Software') == header_server 44 ), 'server software header' 45 46 date = headers.pop('Date') 47 assert date[-4:] == ' GMT', 'date header timezone' 48 assert ( 49 abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5 50 ), 'date header' 51 52 assert headers == { 53 'Connection': 'close', 54 'Content-Length': str(len(body)), 55 'Content-Type': 'text/html', 56 'Request-Method': 'POST', 57 'Request-Uri': '/', 58 'Http-Host': 'localhost', 59 'Server-Protocol': 'HTTP/1.1', 60 'Custom-Header': 'blah, Blah, BLAH', 61 'Wsgi-Version': '(1, 0)', 62 'Wsgi-Url-Scheme': 'http', 63 'Wsgi-Multithread': 'False', 64 'Wsgi-Multiprocess': 'True', 65 'Wsgi-Run-Once': 'False', 66 }, 'headers' 67 assert resp['body'] == body, 'body' 68 69 def test_python_application_query_string(self): 70 self.load('query_string') 71 72 resp = self.get(url='/?var1=val1&var2=val2') 73 74 assert ( 75 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 76 ), 'Query-String header' 77 78 def test_python_application_query_string_space(self): 79 self.load('query_string') 80 81 resp = self.get(url='/ ?var1=val1&var2=val2') 82 assert ( 83 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 84 ), 'Query-String space' 85 86 resp = self.get(url='/ %20?var1=val1&var2=val2') 87 assert ( 88 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 89 ), 'Query-String space 2' 90 91 resp = self.get(url='/ %20 ?var1=val1&var2=val2') 92 assert ( 93 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 94 ), 'Query-String space 3' 95 96 resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2') 97 assert ( 98 resp['headers']['Query-String'] == ' var1= val1 & var2=val2' 99 ), 'Query-String space 4' 100 101 def test_python_application_query_string_empty(self): 102 self.load('query_string') 103 104 resp = self.get(url='/?') 105 106 assert resp['status'] == 200, 'query string empty status' 107 assert resp['headers']['Query-String'] == '', 'query string empty' 108 109 def test_python_application_query_string_absent(self): 110 self.load('query_string') 111 112 resp = self.get() 113 114 assert resp['status'] == 200, 'query string absent status' 115 assert resp['headers']['Query-String'] == '', 'query string absent' 116 117 @pytest.mark.skip('not yet') 118 def test_python_application_server_port(self): 119 self.load('server_port') 120 121 assert ( 122 self.get()['headers']['Server-Port'] == '7080' 123 ), 'Server-Port header' 124 125 @pytest.mark.skip('not yet') 126 def test_python_application_working_directory_invalid(self): 127 self.load('empty') 128 129 assert 'success' in self.conf( 130 '"/blah"', 'applications/empty/working_directory' 131 ), 'configure invalid working_directory' 132 133 assert self.get()['status'] == 500, 'status' 134 135 def test_python_application_204_transfer_encoding(self): 136 self.load('204_no_content') 137 138 assert ( 139 'Transfer-Encoding' not in self.get()['headers'] 140 ), '204 header transfer encoding' 141 142 def test_python_application_ctx_iter_atexit(self): 143 self.load('ctx_iter_atexit') 144 145 resp = self.post( 146 headers={ 147 'Host': 'localhost', 148 'Connection': 'close', 149 'Content-Type': 'text/html', 150 }, 151 body='0123456789', 152 ) 153 154 assert resp['status'] == 200, 'ctx iter status' 155 assert resp['body'] == '0123456789', 'ctx iter body' 156 157 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 158 159 assert ( 160 self.wait_for_record(r'RuntimeError') is not None 161 ), 'ctx iter atexit' 162 163 def test_python_keepalive_body(self): 164 self.load('mirror') 165 166 assert self.get()['status'] == 200, 'init' 167 168 body = '0123456789' * 500 169 (resp, sock) = self.post( 170 headers={ 171 'Host': 'localhost', 172 'Connection': 'keep-alive', 173 'Content-Type': 'text/html', 174 }, 175 start=True, 176 body=body, 177 read_timeout=1, 178 ) 179 180 assert resp['body'] == body, 'keep-alive 1' 181 182 body = '0123456789' 183 resp = self.post( 184 headers={ 185 'Host': 'localhost', 186 'Connection': 'close', 187 'Content-Type': 'text/html', 188 }, 189 sock=sock, 190 body=body, 191 ) 192 193 assert resp['body'] == body, 'keep-alive 2' 194 195 def test_python_keepalive_reconfigure(self): 196 self.load('mirror') 197 198 assert self.get()['status'] == 200, 'init' 199 200 body = '0123456789' 201 conns = 3 202 socks = [] 203 204 for i in range(conns): 205 (resp, sock) = self.post( 206 headers={ 207 'Host': 'localhost', 208 'Connection': 'keep-alive', 209 'Content-Type': 'text/html', 210 }, 211 start=True, 212 body=body, 213 read_timeout=1, 214 ) 215 216 assert resp['body'] == body, 'keep-alive open' 217 218 self.load('mirror', processes=i + 1) 219 220 socks.append(sock) 221 222 for i in range(conns): 223 (resp, sock) = self.post( 224 headers={ 225 'Host': 'localhost', 226 'Connection': 'keep-alive', 227 'Content-Type': 'text/html', 228 }, 229 start=True, 230 sock=socks[i], 231 body=body, 232 read_timeout=1, 233 ) 234 235 assert resp['body'] == body, 'keep-alive request' 236 237 self.load('mirror', processes=i + 1) 238 239 for i in range(conns): 240 resp = self.post( 241 headers={ 242 'Host': 'localhost', 243 'Connection': 'close', 244 'Content-Type': 'text/html', 245 }, 246 sock=socks[i], 247 body=body, 248 ) 249 250 assert resp['body'] == body, 'keep-alive close' 251 252 self.load('mirror', processes=i + 1) 253 254 def test_python_keepalive_reconfigure_2(self): 255 self.load('mirror') 256 257 assert self.get()['status'] == 200, 'init' 258 259 body = '0123456789' 260 261 (resp, sock) = self.post( 262 headers={ 263 'Host': 'localhost', 264 'Connection': 'keep-alive', 265 'Content-Type': 'text/html', 266 }, 267 start=True, 268 body=body, 269 read_timeout=1, 270 ) 271 272 assert resp['body'] == body, 'reconfigure 2 keep-alive 1' 273 274 self.load('empty') 275 276 assert self.get()['status'] == 200, 'init' 277 278 (resp, sock) = self.post( 279 headers={ 280 'Host': 'localhost', 281 'Connection': 'close', 282 'Content-Type': 'text/html', 283 }, 284 start=True, 285 sock=sock, 286 body=body, 287 ) 288 289 assert resp['status'] == 200, 'reconfigure 2 keep-alive 2' 290 assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body' 291 292 assert 'success' in self.conf( 293 {"listeners": {}, "applications": {}} 294 ), 'reconfigure 2 clear configuration' 295 296 resp = self.get(sock=sock) 297 298 assert resp == {}, 'reconfigure 2 keep-alive 3' 299 300 def test_python_keepalive_reconfigure_3(self): 301 self.load('empty') 302 303 assert self.get()['status'] == 200, 'init' 304 305 (_, sock) = self.http( 306 b"""GET / HTTP/1.1 307""", 308 start=True, 309 raw=True, 310 no_recv=True, 311 ) 312 313 assert self.get()['status'] == 200 314 315 assert 'success' in self.conf( 316 {"listeners": {}, "applications": {}} 317 ), 'reconfigure 3 clear configuration' 318 319 resp = self.http( 320 b"""Host: localhost 321Connection: close 322 323""", 324 sock=sock, 325 raw=True, 326 ) 327 328 assert resp['status'] == 200, 'reconfigure 3' 329 330 def test_python_atexit(self): 331 self.load('atexit') 332 333 self.get() 334 335 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 336 337 assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' 338 339 def test_python_process_switch(self): 340 self.load('delayed', processes=2) 341 342 self.get( 343 headers={ 344 'Host': 'localhost', 345 'Content-Length': '0', 346 'X-Delay': '5', 347 'Connection': 'close', 348 }, 349 no_recv=True, 350 ) 351 352 headers_delay_1 = { 353 'Connection': 'close', 354 'Host': 'localhost', 355 'Content-Length': '0', 356 'X-Delay': '1', 357 } 358 359 self.get(headers=headers_delay_1, no_recv=True) 360 361 time.sleep(0.5) 362 363 for _ in range(10): 364 self.get(headers=headers_delay_1, no_recv=True) 365 366 self.get(headers=headers_delay_1) 367 368 @pytest.mark.skip('not yet') 369 def test_python_application_start_response_exit(self): 370 self.load('start_response_exit') 371 372 assert self.get()['status'] == 500, 'start response exit' 373 374 def test_python_application_input_iter(self): 375 self.load('input_iter') 376 377 body = '''0123456789 378next line 379 380last line''' 381 382 resp = self.post(body=body) 383 assert resp['body'] == body, 'input iter' 384 assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines' 385 386 def test_python_application_input_readline(self): 387 self.load('input_readline') 388 389 body = '''0123456789 390next line 391 392last line''' 393 394 resp = self.post(body=body) 395 assert resp['body'] == body, 'input readline' 396 assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines' 397 398 def test_python_application_input_readline_size(self): 399 self.load('input_readline_size') 400 401 body = '''0123456789 402next line 403 404last line''' 405 406 assert self.post(body=body)['body'] == body, 'input readline size' 407 assert ( 408 self.post(body='0123')['body'] == '0123' 409 ), 'input readline size less' 410 411 def test_python_application_input_readlines(self): 412 self.load('input_readlines') 413 414 body = '''0123456789 415next line 416 417last line''' 418 419 resp = self.post(body=body) 420 assert resp['body'] == body, 'input readlines' 421 assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines' 422 423 def test_python_application_input_readlines_huge(self): 424 self.load('input_readlines') 425 426 body = ( 427 '''0123456789 abcdefghi 428next line: 0123456789 abcdefghi 429 430last line: 987654321 431''' 432 * 512 433 ) 434 435 assert ( 436 self.post(body=body, read_buffer_size=16384)['body'] == body 437 ), 'input readlines huge' 438 439 def test_python_application_input_read_length(self): 440 self.load('input_read_length') 441 442 body = '0123456789' 443 444 resp = self.post( 445 headers={ 446 'Host': 'localhost', 447 'Input-Length': '5', 448 'Connection': 'close', 449 }, 450 body=body, 451 ) 452 453 assert resp['body'] == body[:5], 'input read length lt body' 454 455 resp = self.post( 456 headers={ 457 'Host': 'localhost', 458 'Input-Length': '15', 459 'Connection': 'close', 460 }, 461 body=body, 462 ) 463 464 assert resp['body'] == body, 'input read length gt body' 465 466 resp = self.post( 467 headers={ 468 'Host': 'localhost', 469 'Input-Length': '0', 470 'Connection': 'close', 471 }, 472 body=body, 473 ) 474 475 assert resp['body'] == '', 'input read length zero' 476 477 resp = self.post( 478 headers={ 479 'Host': 'localhost', 480 'Input-Length': '-1', 481 'Connection': 'close', 482 }, 483 body=body, 484 ) 485 486 assert resp['body'] == body, 'input read length negative' 487 488 @pytest.mark.skip('not yet') 489 def test_python_application_errors_write(self): 490 self.load('errors_write') 491 492 self.get() 493 494 assert ( 495 self.wait_for_record(r'\[error\].+Error in application\.') 496 is not None 497 ), 'errors write' 498 499 def test_python_application_body_array(self): 500 self.load('body_array') 501 502 assert self.get()['body'] == '0123456789', 'body array' 503 504 def test_python_application_body_io(self): 505 self.load('body_io') 506 507 assert self.get()['body'] == '0123456789', 'body io' 508 509 def test_python_application_body_io_file(self): 510 self.load('body_io_file') 511 512 assert self.get()['body'] == 'body\n', 'body io file' 513 514 @pytest.mark.skip('not yet') 515 def test_python_application_syntax_error(self, skip_alert): 516 skip_alert(r'Python failed to import module "wsgi"') 517 self.load('syntax_error') 518 519 assert self.get()['status'] == 500, 'syntax error' 520 521 def test_python_application_loading_error(self, skip_alert): 522 skip_alert(r'Python failed to import module "blah"') 523 524 self.load('empty', module="blah") 525 526 assert self.get()['status'] == 503, 'loading error' 527 528 def test_python_application_close(self): 529 self.load('close') 530 531 self.get() 532 533 assert self.wait_for_record(r'Close called\.') is not None, 'close' 534 535 def test_python_application_close_error(self): 536 self.load('close_error') 537 538 self.get() 539 540 assert ( 541 self.wait_for_record(r'Close called\.') is not None 542 ), 'close error' 543 544 def test_python_application_not_iterable(self): 545 self.load('not_iterable') 546 547 self.get() 548 549 assert ( 550 self.wait_for_record( 551 r'\[error\].+the application returned not an iterable object' 552 ) 553 is not None 554 ), 'not iterable' 555 556 def test_python_application_write(self): 557 self.load('write') 558 559 assert self.get()['body'] == '0123456789', 'write' 560 561 def test_python_application_threading(self): 562 """wait_for_record() timeouts after 5s while every thread works at 563 least 3s. So without releasing GIL test should fail. 564 """ 565 566 self.load('threading') 567 568 for _ in range(10): 569 self.get(no_recv=True) 570 571 assert ( 572 self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None 573 ), 'last thread finished' 574 575 def test_python_application_iter_exception(self): 576 self.load('iter_exception') 577 578 # Default request doesn't lead to the exception. 579 580 resp = self.get( 581 headers={ 582 'Host': 'localhost', 583 'X-Skip': '9', 584 'X-Chunked': '1', 585 'Connection': 'close', 586 } 587 ) 588 assert resp['status'] == 200, 'status' 589 assert resp['body'] == 'XXXXXXX', 'body' 590 591 # Exception before start_response(). 592 593 assert self.get()['status'] == 503, 'error' 594 595 assert self.wait_for_record(r'Traceback') is not None, 'traceback' 596 assert ( 597 self.wait_for_record(r'raise Exception\(\'first exception\'\)') 598 is not None 599 ), 'first exception raise' 600 assert len(self.findall(r'Traceback')) == 1, 'traceback count 1' 601 602 # Exception after start_response(), before first write(). 603 604 assert ( 605 self.get( 606 headers={ 607 'Host': 'localhost', 608 'X-Skip': '1', 609 'Connection': 'close', 610 } 611 )['status'] 612 == 503 613 ), 'error 2' 614 615 assert ( 616 self.wait_for_record(r'raise Exception\(\'second exception\'\)') 617 is not None 618 ), 'exception raise second' 619 assert len(self.findall(r'Traceback')) == 2, 'traceback count 2' 620 621 # Exception after first write(), before first __next__(). 622 623 _, sock = self.get( 624 headers={ 625 'Host': 'localhost', 626 'X-Skip': '2', 627 'Connection': 'keep-alive', 628 }, 629 start=True, 630 ) 631 632 assert ( 633 self.wait_for_record(r'raise Exception\(\'third exception\'\)') 634 is not None 635 ), 'exception raise third' 636 assert len(self.findall(r'Traceback')) == 3, 'traceback count 3' 637 638 assert self.get(sock=sock) == {}, 'closed connection' 639 640 # Exception after first write(), before first __next__(), 641 # chunked (incomplete body). 642 643 resp = self.get( 644 headers={ 645 'Host': 'localhost', 646 'X-Skip': '2', 647 'X-Chunked': '1', 648 'Connection': 'close', 649 }, 650 raw_resp=True, 651 ) 652 if resp: 653 assert resp[-5:] != '0\r\n\r\n', 'incomplete body' 654 assert len(self.findall(r'Traceback')) == 4, 'traceback count 4' 655 656 # Exception in __next__(). 657 658 _, sock = self.get( 659 headers={ 660 'Host': 'localhost', 661 'X-Skip': '3', 662 'Connection': 'keep-alive', 663 }, 664 start=True, 665 ) 666 667 assert ( 668 self.wait_for_record(r'raise Exception\(\'next exception\'\)') 669 is not None 670 ), 'exception raise next' 671 assert len(self.findall(r'Traceback')) == 5, 'traceback count 5' 672 673 assert self.get(sock=sock) == {}, 'closed connection 2' 674 675 # Exception in __next__(), chunked (incomplete body). 676 677 resp = self.get( 678 headers={ 679 'Host': 'localhost', 680 'X-Skip': '3', 681 'X-Chunked': '1', 682 'Connection': 'close', 683 }, 684 raw_resp=True, 685 ) 686 if resp: 687 assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2' 688 assert len(self.findall(r'Traceback')) == 6, 'traceback count 6' 689 690 # Exception before start_response() and in close(). 691 692 assert ( 693 self.get( 694 headers={ 695 'Host': 'localhost', 696 'X-Not-Skip-Close': '1', 697 'Connection': 'close', 698 } 699 )['status'] 700 == 503 701 ), 'error' 702 703 assert ( 704 self.wait_for_record(r'raise Exception\(\'close exception\'\)') 705 is not None 706 ), 'exception raise close' 707 assert len(self.findall(r'Traceback')) == 8, 'traceback count 8' 708 709 def test_python_user_group(self, is_su): 710 if not is_su: 711 pytest.skip('requires root') 712 713 nobody_uid = pwd.getpwnam('nobody').pw_uid 714 715 group = 'nobody' 716 717 try: 718 group_id = grp.getgrnam(group).gr_gid 719 except KeyError: 720 group = 'nogroup' 721 group_id = grp.getgrnam(group).gr_gid 722 723 self.load('user_group') 724 725 obj = self.getjson()['body'] 726 assert obj['UID'] == nobody_uid, 'nobody uid' 727 assert obj['GID'] == group_id, 'nobody gid' 728 729 self.load('user_group', user='nobody') 730 731 obj = self.getjson()['body'] 732 assert obj['UID'] == nobody_uid, 'nobody uid user=nobody' 733 assert obj['GID'] == group_id, 'nobody gid user=nobody' 734 735 self.load('user_group', user='nobody', group=group) 736 737 obj = self.getjson()['body'] 738 assert obj['UID'] == nobody_uid, ( 739 'nobody uid user=nobody group=%s' % group 740 ) 741 742 assert obj['GID'] == group_id, ( 743 'nobody gid user=nobody group=%s' % group 744 ) 745 746 self.load('user_group', group=group) 747 748 obj = self.getjson()['body'] 749 assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group 750 751 assert obj['GID'] == group_id, 'nobody gid group=%s' % group 752 753 self.load('user_group', user='root') 754 755 obj = self.getjson()['body'] 756 assert obj['UID'] == 0, 'root uid user=root' 757 assert obj['GID'] == 0, 'root gid user=root' 758 759 group = 'root' 760 761 try: 762 grp.getgrnam(group) 763 group = True 764 except KeyError: 765 group = False 766 767 if group: 768 self.load('user_group', user='root', group='root') 769 770 obj = self.getjson()['body'] 771 assert obj['UID'] == 0, 'root uid user=root group=root' 772 assert obj['GID'] == 0, 'root gid user=root group=root' 773 774 self.load('user_group', group='root') 775 776 obj = self.getjson()['body'] 777 assert obj['UID'] == nobody_uid, 'root uid group=root' 778 assert obj['GID'] == 0, 'root gid group=root' 779 780 def test_python_application_callable(self, skip_alert): 781 skip_alert(r'Python failed to get "blah" from module') 782 self.load('callable') 783 784 assert self.get()['status'] == 204, 'default application response' 785 786 self.load('callable', callable="app") 787 788 assert self.get()['status'] == 200, 'callable response' 789 790 self.load('callable', callable="blah") 791 792 assert self.get()['status'] not in [200, 204], 'callable response inv' 793 794 def test_python_application_path(self): 795 self.load('path') 796 797 def set_path(path): 798 assert 'success' in self.conf(path, 'applications/path/path') 799 800 def get_path(): 801 return self.get()['body'].split(os.pathsep) 802 803 default_path = self.conf_get('/config/applications/path/path') 804 assert 'success' in self.conf( 805 {"PYTHONPATH": default_path}, 806 '/config/applications/path/environment', 807 ) 808 809 self.conf_delete('/config/applications/path/path') 810 sys_path = get_path() 811 812 set_path('"/blah"') 813 assert ['/blah', *sys_path] == get_path(), 'check path' 814 815 set_path('"/new"') 816 assert ['/new', *sys_path] == get_path(), 'check path update' 817 818 set_path('["/blah1", "/blah2"]') 819 assert ['/blah1', '/blah2', *sys_path] == get_path(), 'check path array' 820 821 def test_python_application_path_invalid(self): 822 self.load('path') 823 824 def check_path(path): 825 assert 'error' in self.conf(path, 'applications/path/path') 826 827 check_path('{}') 828 check_path('["/blah", []]') 829 830 def test_python_application_threads(self): 831 self.load('threads', threads=4) 832 833 socks = [] 834 835 for i in range(4): 836 (_, sock) = self.get( 837 headers={ 838 'Host': 'localhost', 839 'X-Delay': '2', 840 'Connection': 'close', 841 }, 842 no_recv=True, 843 start=True, 844 ) 845 846 socks.append(sock) 847 848 threads = set() 849 850 for sock in socks: 851 resp = self.recvall(sock).decode('utf-8') 852 853 self.log_in(resp) 854 855 resp = self._resp_to_dict(resp) 856 857 assert resp['status'] == 200, 'status' 858 859 threads.add(resp['headers']['X-Thread']) 860 861 assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' 862 863 sock.close() 864 865 assert len(socks) == len(threads), 'threads differs' 866