1import grp 2import pwd 3import re 4import time 5 6import pytest 7 8from conftest import skip_alert 9from conftest import unit_stop 10from unit.applications.lang.python import TestApplicationPython 11from unit.option import option 12 13 14class TestPythonApplication(TestApplicationPython): 15 prerequisites = {'modules': {'python': 'all'}} 16 17 def findall(self, pattern): 18 with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: 19 return re.findall(pattern, f.read()) 20 21 def test_python_application_variables(self): 22 self.load('variables') 23 24 body = 'Test body string.' 25 26 resp = self.http( 27 b"""POST / HTTP/1.1 28Host: localhost 29Content-Length: %d 30Custom-Header: blah 31Custom-hEader: Blah 32Content-Type: text/html 33Connection: close 34custom-header: BLAH 35 36%s""" % (len(body), body.encode()), 37 raw=True, 38 ) 39 40 assert resp['status'] == 200, 'status' 41 headers = resp['headers'] 42 header_server = headers.pop('Server') 43 assert re.search(r'Unit/[\d\.]+', header_server), 'server header' 44 assert ( 45 headers.pop('Server-Software') == header_server 46 ), 'server software header' 47 48 date = headers.pop('Date') 49 assert date[-4:] == ' GMT', 'date header timezone' 50 assert ( 51 abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5 52 ), 'date header' 53 54 assert headers == { 55 'Connection': 'close', 56 'Content-Length': str(len(body)), 57 'Content-Type': 'text/html', 58 'Request-Method': 'POST', 59 'Request-Uri': '/', 60 'Http-Host': 'localhost', 61 'Server-Protocol': 'HTTP/1.1', 62 'Custom-Header': 'blah, Blah, BLAH', 63 'Wsgi-Version': '(1, 0)', 64 'Wsgi-Url-Scheme': 'http', 65 'Wsgi-Multithread': 'False', 66 'Wsgi-Multiprocess': 'True', 67 'Wsgi-Run-Once': 'False', 68 }, 'headers' 69 assert resp['body'] == body, 'body' 70 71 def test_python_application_query_string(self): 72 self.load('query_string') 73 74 resp = self.get(url='/?var1=val1&var2=val2') 75 76 assert ( 77 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 78 ), 'Query-String header' 79 80 def test_python_application_query_string_space(self): 81 self.load('query_string') 82 83 resp = self.get(url='/ ?var1=val1&var2=val2') 84 assert ( 85 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 86 ), 'Query-String space' 87 88 resp = self.get(url='/ %20?var1=val1&var2=val2') 89 assert ( 90 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 91 ), 'Query-String space 2' 92 93 resp = self.get(url='/ %20 ?var1=val1&var2=val2') 94 assert ( 95 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 96 ), 'Query-String space 3' 97 98 resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2') 99 assert ( 100 resp['headers']['Query-String'] == ' var1= val1 & var2=val2' 101 ), 'Query-String space 4' 102 103 def test_python_application_query_string_empty(self): 104 self.load('query_string') 105 106 resp = self.get(url='/?') 107 108 assert resp['status'] == 200, 'query string empty status' 109 assert resp['headers']['Query-String'] == '', 'query string empty' 110 111 def test_python_application_query_string_absent(self): 112 self.load('query_string') 113 114 resp = self.get() 115 116 assert resp['status'] == 200, 'query string absent status' 117 assert resp['headers']['Query-String'] == '', 'query string absent' 118 119 @pytest.mark.skip('not yet') 120 def test_python_application_server_port(self): 121 self.load('server_port') 122 123 assert ( 124 self.get()['headers']['Server-Port'] == '7080' 125 ), 'Server-Port header' 126 127 @pytest.mark.skip('not yet') 128 def test_python_application_working_directory_invalid(self): 129 self.load('empty') 130 131 assert 'success' in self.conf( 132 '"/blah"', 'applications/empty/working_directory' 133 ), 'configure invalid working_directory' 134 135 assert self.get()['status'] == 500, 'status' 136 137 def test_python_application_204_transfer_encoding(self): 138 self.load('204_no_content') 139 140 assert ( 141 'Transfer-Encoding' not in self.get()['headers'] 142 ), '204 header transfer encoding' 143 144 def test_python_application_ctx_iter_atexit(self): 145 self.load('ctx_iter_atexit') 146 147 resp = self.post( 148 headers={ 149 'Host': 'localhost', 150 'Connection': 'close', 151 'Content-Type': 'text/html', 152 }, 153 body='0123456789', 154 ) 155 156 assert resp['status'] == 200, 'ctx iter status' 157 assert resp['body'] == '0123456789', 'ctx iter body' 158 159 self.conf({"listeners": {}, "applications": {}}) 160 161 unit_stop() 162 163 assert ( 164 self.wait_for_record(r'RuntimeError') is not None 165 ), 'ctx iter atexit' 166 167 def test_python_keepalive_body(self): 168 self.load('mirror') 169 170 assert self.get()['status'] == 200, 'init' 171 172 body = '0123456789' * 500 173 (resp, sock) = self.post( 174 headers={ 175 'Host': 'localhost', 176 'Connection': 'keep-alive', 177 'Content-Type': 'text/html', 178 }, 179 start=True, 180 body=body, 181 read_timeout=1, 182 ) 183 184 assert resp['body'] == body, 'keep-alive 1' 185 186 body = '0123456789' 187 resp = self.post( 188 headers={ 189 'Host': 'localhost', 190 'Connection': 'close', 191 'Content-Type': 'text/html', 192 }, 193 sock=sock, 194 body=body, 195 ) 196 197 assert resp['body'] == body, 'keep-alive 2' 198 199 def test_python_keepalive_reconfigure(self): 200 self.load('mirror') 201 202 assert self.get()['status'] == 200, 'init' 203 204 body = '0123456789' 205 conns = 3 206 socks = [] 207 208 for i in range(conns): 209 (resp, sock) = self.post( 210 headers={ 211 'Host': 'localhost', 212 'Connection': 'keep-alive', 213 'Content-Type': 'text/html', 214 }, 215 start=True, 216 body=body, 217 read_timeout=1, 218 ) 219 220 assert resp['body'] == body, 'keep-alive open' 221 222 self.load('mirror', processes=i + 1) 223 224 socks.append(sock) 225 226 for i in range(conns): 227 (resp, sock) = self.post( 228 headers={ 229 'Host': 'localhost', 230 'Connection': 'keep-alive', 231 'Content-Type': 'text/html', 232 }, 233 start=True, 234 sock=socks[i], 235 body=body, 236 read_timeout=1, 237 ) 238 239 assert resp['body'] == body, 'keep-alive request' 240 241 self.load('mirror', processes=i + 1) 242 243 for i in range(conns): 244 resp = self.post( 245 headers={ 246 'Host': 'localhost', 247 'Connection': 'close', 248 'Content-Type': 'text/html', 249 }, 250 sock=socks[i], 251 body=body, 252 ) 253 254 assert resp['body'] == body, 'keep-alive close' 255 256 self.load('mirror', processes=i + 1) 257 258 def test_python_keepalive_reconfigure_2(self): 259 self.load('mirror') 260 261 assert self.get()['status'] == 200, 'init' 262 263 body = '0123456789' 264 265 (resp, sock) = self.post( 266 headers={ 267 'Host': 'localhost', 268 'Connection': 'keep-alive', 269 'Content-Type': 'text/html', 270 }, 271 start=True, 272 body=body, 273 read_timeout=1, 274 ) 275 276 assert resp['body'] == body, 'reconfigure 2 keep-alive 1' 277 278 self.load('empty') 279 280 assert self.get()['status'] == 200, 'init' 281 282 (resp, sock) = self.post( 283 headers={ 284 'Host': 'localhost', 285 'Connection': 'close', 286 'Content-Type': 'text/html', 287 }, 288 start=True, 289 sock=sock, 290 body=body, 291 ) 292 293 assert resp['status'] == 200, 'reconfigure 2 keep-alive 2' 294 assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body' 295 296 assert 'success' in self.conf( 297 {"listeners": {}, "applications": {}} 298 ), 'reconfigure 2 clear configuration' 299 300 resp = self.get(sock=sock) 301 302 assert resp == {}, 'reconfigure 2 keep-alive 3' 303 304 def test_python_keepalive_reconfigure_3(self): 305 self.load('empty') 306 307 assert self.get()['status'] == 200, 'init' 308 309 (_, sock) = self.http( 310 b"""GET / HTTP/1.1 311""", 312 start=True, 313 raw=True, 314 no_recv=True, 315 ) 316 317 assert self.get()['status'] == 200 318 319 assert 'success' in self.conf( 320 {"listeners": {}, "applications": {}} 321 ), 'reconfigure 3 clear configuration' 322 323 resp = self.http( 324 b"""Host: localhost 325Connection: close 326 327""", 328 sock=sock, 329 raw=True, 330 ) 331 332 assert resp['status'] == 200, 'reconfigure 3' 333 334 def test_python_atexit(self): 335 self.load('atexit') 336 337 self.get() 338 339 self.conf({"listeners": {}, "applications": {}}) 340 341 unit_stop() 342 343 assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' 344 345 def test_python_process_switch(self): 346 self.load('delayed', processes=2) 347 348 self.get( 349 headers={ 350 'Host': 'localhost', 351 'Content-Length': '0', 352 'X-Delay': '5', 353 'Connection': 'close', 354 }, 355 no_recv=True, 356 ) 357 358 headers_delay_1 = { 359 'Connection': 'close', 360 'Host': 'localhost', 361 'Content-Length': '0', 362 'X-Delay': '1', 363 } 364 365 self.get(headers=headers_delay_1, no_recv=True) 366 367 time.sleep(0.5) 368 369 for _ in range(10): 370 self.get(headers=headers_delay_1, no_recv=True) 371 372 self.get(headers=headers_delay_1) 373 374 @pytest.mark.skip('not yet') 375 def test_python_application_start_response_exit(self): 376 self.load('start_response_exit') 377 378 assert self.get()['status'] == 500, 'start response exit' 379 380 def test_python_application_input_iter(self): 381 self.load('input_iter') 382 383 body = '''0123456789 384next line 385 386last line''' 387 388 resp = self.post(body=body) 389 assert resp['body'] == body, 'input iter' 390 assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines' 391 392 def test_python_application_input_readline(self): 393 self.load('input_readline') 394 395 body = '''0123456789 396next line 397 398last line''' 399 400 resp = self.post(body=body) 401 assert resp['body'] == body, 'input readline' 402 assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines' 403 404 def test_python_application_input_readline_size(self): 405 self.load('input_readline_size') 406 407 body = '''0123456789 408next line 409 410last line''' 411 412 assert self.post(body=body)['body'] == body, 'input readline size' 413 assert ( 414 self.post(body='0123')['body'] == '0123' 415 ), 'input readline size less' 416 417 def test_python_application_input_readlines(self): 418 self.load('input_readlines') 419 420 body = '''0123456789 421next line 422 423last line''' 424 425 resp = self.post(body=body) 426 assert resp['body'] == body, 'input readlines' 427 assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines' 428 429 def test_python_application_input_readlines_huge(self): 430 self.load('input_readlines') 431 432 body = ( 433 '''0123456789 abcdefghi 434next line: 0123456789 abcdefghi 435 436last line: 987654321 437''' 438 * 512 439 ) 440 441 assert ( 442 self.post(body=body, read_buffer_size=16384)['body'] == body 443 ), 'input readlines huge' 444 445 def test_python_application_input_read_length(self): 446 self.load('input_read_length') 447 448 body = '0123456789' 449 450 resp = self.post( 451 headers={ 452 'Host': 'localhost', 453 'Input-Length': '5', 454 'Connection': 'close', 455 }, 456 body=body, 457 ) 458 459 assert resp['body'] == body[:5], 'input read length lt body' 460 461 resp = self.post( 462 headers={ 463 'Host': 'localhost', 464 'Input-Length': '15', 465 'Connection': 'close', 466 }, 467 body=body, 468 ) 469 470 assert resp['body'] == body, 'input read length gt body' 471 472 resp = self.post( 473 headers={ 474 'Host': 'localhost', 475 'Input-Length': '0', 476 'Connection': 'close', 477 }, 478 body=body, 479 ) 480 481 assert resp['body'] == '', 'input read length zero' 482 483 resp = self.post( 484 headers={ 485 'Host': 'localhost', 486 'Input-Length': '-1', 487 'Connection': 'close', 488 }, 489 body=body, 490 ) 491 492 assert resp['body'] == body, 'input read length negative' 493 494 @pytest.mark.skip('not yet') 495 def test_python_application_errors_write(self): 496 self.load('errors_write') 497 498 self.get() 499 500 unit_stop() 501 502 assert ( 503 self.wait_for_record(r'\[error\].+Error in application\.') 504 is not None 505 ), 'errors write' 506 507 def test_python_application_body_array(self): 508 self.load('body_array') 509 510 assert self.get()['body'] == '0123456789', 'body array' 511 512 def test_python_application_body_io(self): 513 self.load('body_io') 514 515 assert self.get()['body'] == '0123456789', 'body io' 516 517 def test_python_application_body_io_file(self): 518 self.load('body_io_file') 519 520 assert self.get()['body'] == 'body\n', 'body io file' 521 522 @pytest.mark.skip('not yet') 523 def test_python_application_syntax_error(self): 524 skip_alert(r'Python failed to import module "wsgi"') 525 self.load('syntax_error') 526 527 assert self.get()['status'] == 500, 'syntax error' 528 529 def test_python_application_loading_error(self): 530 skip_alert(r'Python failed to import module "blah"') 531 532 self.load('empty', module="blah") 533 534 assert self.get()['status'] == 503, 'loading error' 535 536 def test_python_application_close(self): 537 self.load('close') 538 539 self.get() 540 541 unit_stop() 542 543 assert self.wait_for_record(r'Close called\.') is not None, 'close' 544 545 def test_python_application_close_error(self): 546 self.load('close_error') 547 548 self.get() 549 550 unit_stop() 551 552 assert ( 553 self.wait_for_record(r'Close called\.') is not None 554 ), 'close error' 555 556 def test_python_application_not_iterable(self): 557 self.load('not_iterable') 558 559 self.get() 560 561 unit_stop() 562 563 assert ( 564 self.wait_for_record( 565 r'\[error\].+the application returned not an iterable object' 566 ) 567 is not None 568 ), 'not iterable' 569 570 def test_python_application_write(self): 571 self.load('write') 572 573 assert self.get()['body'] == '0123456789', 'write' 574 575 def test_python_application_threading(self): 576 """wait_for_record() timeouts after 5s while every thread works at 577 least 3s. So without releasing GIL test should fail. 578 """ 579 580 self.load('threading') 581 582 for _ in range(10): 583 self.get(no_recv=True) 584 585 assert ( 586 self.wait_for_record(r'\(5\) Thread: 100') is not None 587 ), 'last thread finished' 588 589 def test_python_application_iter_exception(self): 590 self.load('iter_exception') 591 592 # Default request doesn't lead to the exception. 593 594 resp = self.get( 595 headers={ 596 'Host': 'localhost', 597 'X-Skip': '9', 598 'X-Chunked': '1', 599 'Connection': 'close', 600 } 601 ) 602 assert resp['status'] == 200, 'status' 603 assert resp['body'] == 'XXXXXXX', 'body' 604 605 # Exception before start_response(). 606 607 assert self.get()['status'] == 503, 'error' 608 609 assert self.wait_for_record(r'Traceback') is not None, 'traceback' 610 assert ( 611 self.wait_for_record(r'raise Exception\(\'first exception\'\)') 612 is not None 613 ), 'first exception raise' 614 assert len(self.findall(r'Traceback')) == 1, 'traceback count 1' 615 616 # Exception after start_response(), before first write(). 617 618 assert ( 619 self.get( 620 headers={ 621 'Host': 'localhost', 622 'X-Skip': '1', 623 'Connection': 'close', 624 } 625 )['status'] 626 == 503 627 ), 'error 2' 628 629 assert ( 630 self.wait_for_record(r'raise Exception\(\'second exception\'\)') 631 is not None 632 ), 'exception raise second' 633 assert len(self.findall(r'Traceback')) == 2, 'traceback count 2' 634 635 # Exception after first write(), before first __next__(). 636 637 _, sock = self.get( 638 headers={ 639 'Host': 'localhost', 640 'X-Skip': '2', 641 'Connection': 'keep-alive', 642 }, 643 start=True, 644 ) 645 646 assert ( 647 self.wait_for_record(r'raise Exception\(\'third exception\'\)') 648 is not None 649 ), 'exception raise third' 650 assert len(self.findall(r'Traceback')) == 3, 'traceback count 3' 651 652 assert self.get(sock=sock) == {}, 'closed connection' 653 654 # Exception after first write(), before first __next__(), 655 # chunked (incomplete body). 656 657 resp = self.get( 658 headers={ 659 'Host': 'localhost', 660 'X-Skip': '2', 661 'X-Chunked': '1', 662 'Connection': 'close', 663 }, 664 raw_resp=True, 665 ) 666 if resp: 667 assert resp[-5:] != '0\r\n\r\n', 'incomplete body' 668 assert len(self.findall(r'Traceback')) == 4, 'traceback count 4' 669 670 # Exception in __next__(). 671 672 _, sock = self.get( 673 headers={ 674 'Host': 'localhost', 675 'X-Skip': '3', 676 'Connection': 'keep-alive', 677 }, 678 start=True, 679 ) 680 681 assert ( 682 self.wait_for_record(r'raise Exception\(\'next exception\'\)') 683 is not None 684 ), 'exception raise next' 685 assert len(self.findall(r'Traceback')) == 5, 'traceback count 5' 686 687 assert self.get(sock=sock) == {}, 'closed connection 2' 688 689 # Exception in __next__(), chunked (incomplete body). 690 691 resp = self.get( 692 headers={ 693 'Host': 'localhost', 694 'X-Skip': '3', 695 'X-Chunked': '1', 696 'Connection': 'close', 697 }, 698 raw_resp=True, 699 ) 700 if resp: 701 assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2' 702 assert len(self.findall(r'Traceback')) == 6, 'traceback count 6' 703 704 # Exception before start_response() and in close(). 705 706 assert ( 707 self.get( 708 headers={ 709 'Host': 'localhost', 710 'X-Not-Skip-Close': '1', 711 'Connection': 'close', 712 } 713 )['status'] 714 == 503 715 ), 'error' 716 717 assert ( 718 self.wait_for_record(r'raise Exception\(\'close exception\'\)') 719 is not None 720 ), 'exception raise close' 721 assert len(self.findall(r'Traceback')) == 8, 'traceback count 8' 722 723 def test_python_user_group(self, is_su): 724 if not is_su: 725 pytest.skip('requires root') 726 727 nobody_uid = pwd.getpwnam('nobody').pw_uid 728 729 group = 'nobody' 730 731 try: 732 group_id = grp.getgrnam(group).gr_gid 733 except KeyError: 734 group = 'nogroup' 735 group_id = grp.getgrnam(group).gr_gid 736 737 self.load('user_group') 738 739 obj = self.getjson()['body'] 740 assert obj['UID'] == nobody_uid, 'nobody uid' 741 assert obj['GID'] == group_id, 'nobody gid' 742 743 self.load('user_group', user='nobody') 744 745 obj = self.getjson()['body'] 746 assert obj['UID'] == nobody_uid, 'nobody uid user=nobody' 747 assert obj['GID'] == group_id, 'nobody gid user=nobody' 748 749 self.load('user_group', user='nobody', group=group) 750 751 obj = self.getjson()['body'] 752 assert obj['UID'] == nobody_uid, ( 753 'nobody uid user=nobody group=%s' % group 754 ) 755 756 assert obj['GID'] == group_id, ( 757 'nobody gid user=nobody group=%s' % group 758 ) 759 760 self.load('user_group', group=group) 761 762 obj = self.getjson()['body'] 763 assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group 764 765 assert obj['GID'] == group_id, 'nobody gid group=%s' % group 766 767 self.load('user_group', user='root') 768 769 obj = self.getjson()['body'] 770 assert obj['UID'] == 0, 'root uid user=root' 771 assert obj['GID'] == 0, 'root gid user=root' 772 773 group = 'root' 774 775 try: 776 grp.getgrnam(group) 777 group = True 778 except KeyError: 779 group = False 780 781 if group: 782 self.load('user_group', user='root', group='root') 783 784 obj = self.getjson()['body'] 785 assert obj['UID'] == 0, 'root uid user=root group=root' 786 assert obj['GID'] == 0, 'root gid user=root group=root' 787 788 self.load('user_group', group='root') 789 790 obj = self.getjson()['body'] 791 assert obj['UID'] == nobody_uid, 'root uid group=root' 792 assert obj['GID'] == 0, 'root gid group=root' 793 794 def test_python_application_callable(self): 795 skip_alert(r'Python failed to get "blah" from module') 796 self.load('callable') 797 798 assert self.get()['status'] == 204, 'default application response' 799 800 self.load('callable', callable="app") 801 802 assert self.get()['status'] == 200, 'callable response' 803 804 self.load('callable', callable="blah") 805 806 assert self.get()['status'] not in [200, 204], 'callable response inv' 807 808 def test_python_application_threads(self): 809 self.load('threads', threads=4) 810 811 socks = [] 812 813 for i in range(4): 814 (_, sock) = self.get( 815 headers={ 816 'Host': 'localhost', 817 'X-Delay': '2', 818 'Connection': 'close', 819 }, 820 no_recv=True, 821 start=True, 822 ) 823 824 socks.append(sock) 825 826 threads = set() 827 828 for sock in socks: 829 resp = self.recvall(sock).decode('utf-8') 830 831 self.log_in(resp) 832 833 resp = self._resp_to_dict(resp) 834 835 assert resp['status'] == 200, 'status' 836 837 threads.add(resp['headers']['X-Thread']) 838 839 assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' 840 841 sock.close() 842 843 assert len(socks) == len(threads), 'threads differs' 844