1import grp 2import os 3import pwd 4import re 5import time 6 7import pytest 8from unit.applications.lang.python import TestApplicationPython 9 10 11class TestPythonApplication(TestApplicationPython): 12 prerequisites = {'modules': {'python': 'all'}} 13 14 def test_python_application_variables(self): 15 self.load('variables') 16 17 body = 'Test body string.' 18 19 resp = self.http( 20 b"""POST / HTTP/1.1 21Host: localhost 22Content-Length: %d 23Custom-Header: blah 24Custom-hEader: Blah 25Content-Type: text/html 26Connection: close 27custom-header: BLAH 28 29%s""" 30 % (len(body), body.encode()), 31 raw=True, 32 ) 33 34 assert resp['status'] == 200, 'status' 35 headers = resp['headers'] 36 header_server = headers.pop('Server') 37 assert re.search(r'Unit/[\d\.]+', header_server), 'server header' 38 assert ( 39 headers.pop('Server-Software') == header_server 40 ), 'server software header' 41 42 date = headers.pop('Date') 43 assert date[-4:] == ' GMT', 'date header timezone' 44 assert ( 45 abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5 46 ), 'date header' 47 48 assert headers == { 49 'Connection': 'close', 50 'Content-Length': str(len(body)), 51 'Content-Type': 'text/html', 52 'Request-Method': 'POST', 53 'Request-Uri': '/', 54 'Http-Host': 'localhost', 55 'Server-Protocol': 'HTTP/1.1', 56 'Custom-Header': 'blah, Blah, BLAH', 57 'Wsgi-Version': '(1, 0)', 58 'Wsgi-Url-Scheme': 'http', 59 'Wsgi-Multithread': 'False', 60 'Wsgi-Multiprocess': 'True', 61 'Wsgi-Run-Once': 'False', 62 }, 'headers' 63 assert resp['body'] == body, 'body' 64 65 def test_python_application_query_string(self): 66 self.load('query_string') 67 68 resp = self.get(url='/?var1=val1&var2=val2') 69 70 assert ( 71 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 72 ), 'Query-String header' 73 74 def test_python_application_query_string_space(self): 75 self.load('query_string') 76 77 resp = self.get(url='/ ?var1=val1&var2=val2') 78 assert ( 79 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 80 ), 'Query-String space' 81 82 resp = self.get(url='/ %20?var1=val1&var2=val2') 83 assert ( 84 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 85 ), 'Query-String space 2' 86 87 resp = self.get(url='/ %20 ?var1=val1&var2=val2') 88 assert ( 89 resp['headers']['Query-String'] == 'var1=val1&var2=val2' 90 ), 'Query-String space 3' 91 92 resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2') 93 assert ( 94 resp['headers']['Query-String'] == ' var1= val1 & var2=val2' 95 ), 'Query-String space 4' 96 97 def test_python_application_query_string_empty(self): 98 self.load('query_string') 99 100 resp = self.get(url='/?') 101 102 assert resp['status'] == 200, 'query string empty status' 103 assert resp['headers']['Query-String'] == '', 'query string empty' 104 105 def test_python_application_query_string_absent(self): 106 self.load('query_string') 107 108 resp = self.get() 109 110 assert resp['status'] == 200, 'query string absent status' 111 assert resp['headers']['Query-String'] == '', 'query string absent' 112 113 @pytest.mark.skip('not yet') 114 def test_python_application_server_port(self): 115 self.load('server_port') 116 117 assert ( 118 self.get()['headers']['Server-Port'] == '7080' 119 ), 'Server-Port header' 120 121 @pytest.mark.skip('not yet') 122 def test_python_application_working_directory_invalid(self): 123 self.load('empty') 124 125 assert 'success' in self.conf( 126 '"/blah"', 'applications/empty/working_directory' 127 ), 'configure invalid working_directory' 128 129 assert self.get()['status'] == 500, 'status' 130 131 def test_python_application_204_transfer_encoding(self): 132 self.load('204_no_content') 133 134 assert ( 135 'Transfer-Encoding' not in self.get()['headers'] 136 ), '204 header transfer encoding' 137 138 def test_python_application_ctx_iter_atexit(self): 139 self.load('ctx_iter_atexit') 140 141 resp = self.post(body='0123456789') 142 143 assert resp['status'] == 200, 'ctx iter status' 144 assert resp['body'] == '0123456789', 'ctx iter body' 145 146 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 147 148 assert ( 149 self.wait_for_record(r'RuntimeError') is not None 150 ), 'ctx iter atexit' 151 152 def test_python_keepalive_body(self): 153 self.load('mirror') 154 155 assert self.get()['status'] == 200, 'init' 156 157 body = '0123456789' * 500 158 (resp, sock) = self.post( 159 headers={ 160 'Host': 'localhost', 161 'Connection': 'keep-alive', 162 }, 163 start=True, 164 body=body, 165 read_timeout=1, 166 ) 167 168 assert resp['body'] == body, 'keep-alive 1' 169 170 body = '0123456789' 171 resp = self.post(sock=sock, body=body) 172 173 assert resp['body'] == body, 'keep-alive 2' 174 175 def test_python_keepalive_reconfigure(self): 176 self.load('mirror') 177 178 assert self.get()['status'] == 200, 'init' 179 180 body = '0123456789' 181 conns = 3 182 socks = [] 183 184 for i in range(conns): 185 (resp, sock) = self.post( 186 headers={ 187 'Host': 'localhost', 188 'Connection': 'keep-alive', 189 }, 190 start=True, 191 body=body, 192 read_timeout=1, 193 ) 194 195 assert resp['body'] == body, 'keep-alive open' 196 197 self.load('mirror', processes=i + 1) 198 199 socks.append(sock) 200 201 for i in range(conns): 202 (resp, sock) = self.post( 203 headers={ 204 'Host': 'localhost', 205 'Connection': 'keep-alive', 206 }, 207 start=True, 208 sock=socks[i], 209 body=body, 210 read_timeout=1, 211 ) 212 213 assert resp['body'] == body, 'keep-alive request' 214 215 self.load('mirror', processes=i + 1) 216 217 for i in range(conns): 218 resp = self.post(sock=socks[i], body=body) 219 220 assert resp['body'] == body, 'keep-alive close' 221 222 self.load('mirror', processes=i + 1) 223 224 def test_python_keepalive_reconfigure_2(self): 225 self.load('mirror') 226 227 assert self.get()['status'] == 200, 'init' 228 229 body = '0123456789' 230 231 (resp, sock) = self.post( 232 headers={ 233 'Host': 'localhost', 234 'Connection': 'keep-alive', 235 }, 236 start=True, 237 body=body, 238 read_timeout=1, 239 ) 240 241 assert resp['body'] == body, 'reconfigure 2 keep-alive 1' 242 243 self.load('empty') 244 245 assert self.get()['status'] == 200, 'init' 246 247 (resp, sock) = self.post(start=True, sock=sock, body=body) 248 249 assert resp['status'] == 200, 'reconfigure 2 keep-alive 2' 250 assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body' 251 252 assert 'success' in self.conf( 253 {"listeners": {}, "applications": {}} 254 ), 'reconfigure 2 clear configuration' 255 256 resp = self.get(sock=sock) 257 258 assert resp == {}, 'reconfigure 2 keep-alive 3' 259 260 def test_python_atexit(self): 261 self.load('atexit') 262 263 self.get() 264 265 assert 'success' in self.conf({"listeners": {}, "applications": {}}) 266 267 assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' 268 269 def test_python_process_switch(self): 270 self.load('delayed', processes=2) 271 272 self.get( 273 headers={ 274 'Host': 'localhost', 275 'Content-Length': '0', 276 'X-Delay': '5', 277 'Connection': 'close', 278 }, 279 no_recv=True, 280 ) 281 282 headers_delay_1 = { 283 'Connection': 'close', 284 'Host': 'localhost', 285 'Content-Length': '0', 286 'X-Delay': '1', 287 } 288 289 self.get(headers=headers_delay_1, no_recv=True) 290 291 time.sleep(0.5) 292 293 for _ in range(10): 294 self.get(headers=headers_delay_1, no_recv=True) 295 296 self.get(headers=headers_delay_1) 297 298 @pytest.mark.skip('not yet') 299 def test_python_application_start_response_exit(self): 300 self.load('start_response_exit') 301 302 assert self.get()['status'] == 500, 'start response exit' 303 304 def test_python_application_input_iter(self): 305 self.load('input_iter') 306 307 body = '''0123456789 308next line 309 310last line''' 311 312 resp = self.post(body=body) 313 assert resp['body'] == body, 'input iter' 314 assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines' 315 316 def test_python_application_input_readline(self): 317 self.load('input_readline') 318 319 body = '''0123456789 320next line 321 322last line''' 323 324 resp = self.post(body=body) 325 assert resp['body'] == body, 'input readline' 326 assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines' 327 328 def test_python_application_input_readline_size(self): 329 self.load('input_readline_size') 330 331 body = '''0123456789 332next line 333 334last line''' 335 336 assert self.post(body=body)['body'] == body, 'input readline size' 337 assert ( 338 self.post(body='0123')['body'] == '0123' 339 ), 'input readline size less' 340 341 def test_python_application_input_readlines(self): 342 self.load('input_readlines') 343 344 body = '''0123456789 345next line 346 347last line''' 348 349 resp = self.post(body=body) 350 assert resp['body'] == body, 'input readlines' 351 assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines' 352 353 def test_python_application_input_readlines_huge(self): 354 self.load('input_readlines') 355 356 body = ( 357 '''0123456789 abcdefghi 358next line: 0123456789 abcdefghi 359 360last line: 987654321 361''' 362 * 512 363 ) 364 365 assert ( 366 self.post(body=body, read_buffer_size=16384)['body'] == body 367 ), 'input readlines huge' 368 369 def test_python_application_input_read_length(self): 370 self.load('input_read_length') 371 372 body = '0123456789' 373 374 resp = self.post( 375 headers={ 376 'Host': 'localhost', 377 'Input-Length': '5', 378 'Connection': 'close', 379 }, 380 body=body, 381 ) 382 383 assert resp['body'] == body[:5], 'input read length lt body' 384 385 resp = self.post( 386 headers={ 387 'Host': 'localhost', 388 'Input-Length': '15', 389 'Connection': 'close', 390 }, 391 body=body, 392 ) 393 394 assert resp['body'] == body, 'input read length gt body' 395 396 resp = self.post( 397 headers={ 398 'Host': 'localhost', 399 'Input-Length': '0', 400 'Connection': 'close', 401 }, 402 body=body, 403 ) 404 405 assert resp['body'] == '', 'input read length zero' 406 407 resp = self.post( 408 headers={ 409 'Host': 'localhost', 410 'Input-Length': '-1', 411 'Connection': 'close', 412 }, 413 body=body, 414 ) 415 416 assert resp['body'] == body, 'input read length negative' 417 418 @pytest.mark.skip('not yet') 419 def test_python_application_errors_write(self): 420 self.load('errors_write') 421 422 self.get() 423 424 assert ( 425 self.wait_for_record(r'\[error\].+Error in application\.') 426 is not None 427 ), 'errors write' 428 429 def test_python_application_body_array(self): 430 self.load('body_array') 431 432 assert self.get()['body'] == '0123456789', 'body array' 433 434 def test_python_application_body_io(self): 435 self.load('body_io') 436 437 assert self.get()['body'] == '0123456789', 'body io' 438 439 def test_python_application_body_io_file(self): 440 self.load('body_io_file') 441 442 assert self.get()['body'] == 'body\n', 'body io file' 443 444 @pytest.mark.skip('not yet') 445 def test_python_application_syntax_error(self, skip_alert): 446 skip_alert(r'Python failed to import module "wsgi"') 447 self.load('syntax_error') 448 449 assert self.get()['status'] == 500, 'syntax error' 450 451 def test_python_application_loading_error(self, skip_alert): 452 skip_alert(r'Python failed to import module "blah"') 453 454 self.load('empty', module="blah") 455 456 assert self.get()['status'] == 503, 'loading error' 457 458 def test_python_application_close(self): 459 self.load('close') 460 461 self.get() 462 463 assert self.wait_for_record(r'Close called\.') is not None, 'close' 464 465 def test_python_application_close_error(self): 466 self.load('close_error') 467 468 self.get() 469 470 assert ( 471 self.wait_for_record(r'Close called\.') is not None 472 ), 'close error' 473 474 def test_python_application_not_iterable(self): 475 self.load('not_iterable') 476 477 self.get() 478 479 assert ( 480 self.wait_for_record( 481 r'\[error\].+the application returned not an iterable object' 482 ) 483 is not None 484 ), 'not iterable' 485 486 def test_python_application_write(self): 487 self.load('write') 488 489 assert self.get()['body'] == '0123456789', 'write' 490 491 def test_python_application_threading(self): 492 """wait_for_record() timeouts after 5s while every thread works at 493 least 3s. So without releasing GIL test should fail. 494 """ 495 496 self.load('threading') 497 498 for _ in range(10): 499 self.get(no_recv=True) 500 501 assert ( 502 self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None 503 ), 'last thread finished' 504 505 def test_python_application_iter_exception(self): 506 self.load('iter_exception') 507 508 # Default request doesn't lead to the exception. 509 510 resp = self.get( 511 headers={ 512 'Host': 'localhost', 513 'X-Skip': '9', 514 'X-Chunked': '1', 515 'Connection': 'close', 516 } 517 ) 518 assert resp['status'] == 200, 'status' 519 assert resp['body'] == 'XXXXXXX', 'body' 520 521 # Exception before start_response(). 522 523 assert self.get()['status'] == 503, 'error' 524 525 assert self.wait_for_record(r'Traceback') is not None, 'traceback' 526 assert ( 527 self.wait_for_record(r'raise Exception\(\'first exception\'\)') 528 is not None 529 ), 'first exception raise' 530 assert len(self.findall(r'Traceback')) == 1, 'traceback count 1' 531 532 # Exception after start_response(), before first write(). 533 534 assert ( 535 self.get( 536 headers={ 537 'Host': 'localhost', 538 'X-Skip': '1', 539 'Connection': 'close', 540 } 541 )['status'] 542 == 503 543 ), 'error 2' 544 545 assert ( 546 self.wait_for_record(r'raise Exception\(\'second exception\'\)') 547 is not None 548 ), 'exception raise second' 549 assert len(self.findall(r'Traceback')) == 2, 'traceback count 2' 550 551 # Exception after first write(), before first __next__(). 552 553 _, sock = self.get( 554 headers={ 555 'Host': 'localhost', 556 'X-Skip': '2', 557 'Connection': 'keep-alive', 558 }, 559 start=True, 560 ) 561 562 assert ( 563 self.wait_for_record(r'raise Exception\(\'third exception\'\)') 564 is not None 565 ), 'exception raise third' 566 assert len(self.findall(r'Traceback')) == 3, 'traceback count 3' 567 568 assert self.get(sock=sock) == {}, 'closed connection' 569 570 # Exception after first write(), before first __next__(), 571 # chunked (incomplete body). 572 573 resp = self.get( 574 headers={ 575 'Host': 'localhost', 576 'X-Skip': '2', 577 'X-Chunked': '1', 578 'Connection': 'close', 579 }, 580 raw_resp=True, 581 ) 582 if resp: 583 assert resp[-5:] != '0\r\n\r\n', 'incomplete body' 584 assert len(self.findall(r'Traceback')) == 4, 'traceback count 4' 585 586 # Exception in __next__(). 587 588 _, sock = self.get( 589 headers={ 590 'Host': 'localhost', 591 'X-Skip': '3', 592 'Connection': 'keep-alive', 593 }, 594 start=True, 595 ) 596 597 assert ( 598 self.wait_for_record(r'raise Exception\(\'next exception\'\)') 599 is not None 600 ), 'exception raise next' 601 assert len(self.findall(r'Traceback')) == 5, 'traceback count 5' 602 603 assert self.get(sock=sock) == {}, 'closed connection 2' 604 605 # Exception in __next__(), chunked (incomplete body). 606 607 resp = self.get( 608 headers={ 609 'Host': 'localhost', 610 'X-Skip': '3', 611 'X-Chunked': '1', 612 'Connection': 'close', 613 }, 614 raw_resp=True, 615 ) 616 if resp: 617 assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2' 618 assert len(self.findall(r'Traceback')) == 6, 'traceback count 6' 619 620 # Exception before start_response() and in close(). 621 622 assert ( 623 self.get( 624 headers={ 625 'Host': 'localhost', 626 'X-Not-Skip-Close': '1', 627 'Connection': 'close', 628 } 629 )['status'] 630 == 503 631 ), 'error' 632 633 assert ( 634 self.wait_for_record(r'raise Exception\(\'close exception\'\)') 635 is not None 636 ), 'exception raise close' 637 assert len(self.findall(r'Traceback')) == 8, 'traceback count 8' 638 639 def test_python_user_group(self, is_su): 640 if not is_su: 641 pytest.skip('requires root') 642 643 nobody_uid = pwd.getpwnam('nobody').pw_uid 644 645 group = 'nobody' 646 647 try: 648 group_id = grp.getgrnam(group).gr_gid 649 except KeyError: 650 group = 'nogroup' 651 group_id = grp.getgrnam(group).gr_gid 652 653 self.load('user_group') 654 655 obj = self.getjson()['body'] 656 assert obj['UID'] == nobody_uid, 'nobody uid' 657 assert obj['GID'] == group_id, 'nobody gid' 658 659 self.load('user_group', user='nobody') 660 661 obj = self.getjson()['body'] 662 assert obj['UID'] == nobody_uid, 'nobody uid user=nobody' 663 assert obj['GID'] == group_id, 'nobody gid user=nobody' 664 665 self.load('user_group', user='nobody', group=group) 666 667 obj = self.getjson()['body'] 668 assert obj['UID'] == nobody_uid, ( 669 'nobody uid user=nobody group=%s' % group 670 ) 671 672 assert obj['GID'] == group_id, 'nobody gid user=nobody group=%s' % group 673 674 self.load('user_group', group=group) 675 676 obj = self.getjson()['body'] 677 assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group 678 679 assert obj['GID'] == group_id, 'nobody gid group=%s' % group 680 681 self.load('user_group', user='root') 682 683 obj = self.getjson()['body'] 684 assert obj['UID'] == 0, 'root uid user=root' 685 assert obj['GID'] == 0, 'root gid user=root' 686 687 group = 'root' 688 689 try: 690 grp.getgrnam(group) 691 group = True 692 except KeyError: 693 group = False 694 695 if group: 696 self.load('user_group', user='root', group='root') 697 698 obj = self.getjson()['body'] 699 assert obj['UID'] == 0, 'root uid user=root group=root' 700 assert obj['GID'] == 0, 'root gid user=root group=root' 701 702 self.load('user_group', group='root') 703 704 obj = self.getjson()['body'] 705 assert obj['UID'] == nobody_uid, 'root uid group=root' 706 assert obj['GID'] == 0, 'root gid group=root' 707 708 def test_python_application_callable(self, skip_alert): 709 skip_alert(r'Python failed to get "blah" from module') 710 self.load('callable') 711 712 assert self.get()['status'] == 204, 'default application response' 713 714 self.load('callable', callable="app") 715 716 assert self.get()['status'] == 200, 'callable response' 717 718 self.load('callable', callable="blah") 719 720 assert self.get()['status'] not in [200, 204], 'callable response inv' 721 722 def test_python_application_path(self): 723 self.load('path') 724 725 def set_path(path): 726 assert 'success' in self.conf(path, 'applications/path/path') 727 728 def get_path(): 729 return self.get()['body'].split(os.pathsep) 730 731 default_path = self.conf_get('/config/applications/path/path') 732 assert 'success' in self.conf( 733 {"PYTHONPATH": default_path}, 734 '/config/applications/path/environment', 735 ) 736 737 self.conf_delete('/config/applications/path/path') 738 sys_path = get_path() 739 740 set_path('"/blah"') 741 assert ['/blah', *sys_path] == get_path(), 'check path' 742 743 set_path('"/new"') 744 assert ['/new', *sys_path] == get_path(), 'check path update' 745 746 set_path('["/blah1", "/blah2"]') 747 assert [ 748 '/blah1', 749 '/blah2', 750 *sys_path, 751 ] == get_path(), 'check path array' 752 753 def test_python_application_path_invalid(self): 754 self.load('path') 755 756 def check_path(path): 757 assert 'error' in self.conf(path, 'applications/path/path') 758 759 check_path('{}') 760 check_path('["/blah", []]') 761 762 def test_python_application_threads(self): 763 self.load('threads', threads=4) 764 765 socks = [] 766 767 for i in range(4): 768 (_, sock) = self.get( 769 headers={ 770 'Host': 'localhost', 771 'X-Delay': '2', 772 'Connection': 'close', 773 }, 774 no_recv=True, 775 start=True, 776 ) 777 778 socks.append(sock) 779 780 threads = set() 781 782 for sock in socks: 783 resp = self.recvall(sock).decode('utf-8') 784 785 self.log_in(resp) 786 787 resp = self._resp_to_dict(resp) 788 789 assert resp['status'] == 200, 'status' 790 791 threads.add(resp['headers']['X-Thread']) 792 793 assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' 794 795 sock.close() 796 797 assert len(socks) == len(threads), 'threads differs' 798