1import fcntl 2import inspect 3import json 4import os 5import platform 6import re 7import shutil 8import signal 9import stat 10import subprocess 11import sys 12import tempfile 13import time 14from multiprocessing import Process 15 16import pytest 17from unit.check.chroot import check_chroot 18from unit.check.go import check_go 19from unit.check.isolation import check_isolation 20from unit.check.njs import check_njs 21from unit.check.node import check_node 22from unit.check.regex import check_regex 23from unit.check.tls import check_openssl 24from unit.check.unix_abstract import check_unix_abstract 25from unit.http import TestHTTP 26from unit.log import Log 27from unit.option import option 28from unit.status import Status 29from unit.utils import public_dir 30from unit.utils import waitforfiles 31 32 33def pytest_addoption(parser): 34 parser.addoption( 35 "--detailed", 36 default=False, 37 action="store_true", 38 help="Detailed output for tests", 39 ) 40 parser.addoption( 41 "--print-log", 42 default=False, 43 action="store_true", 44 help="Print unit.log to stdout in case of errors", 45 ) 46 parser.addoption( 47 "--save-log", 48 default=False, 49 action="store_true", 50 help="Save unit.log after the test execution", 51 ) 52 parser.addoption( 53 "--unsafe", 54 default=False, 55 action="store_true", 56 help="Run unsafe tests", 57 ) 58 parser.addoption( 59 "--user", 60 type=str, 61 help="Default user for non-privileged processes of unitd", 62 ) 63 parser.addoption( 64 "--fds-threshold", 65 type=int, 66 default=0, 67 help="File descriptors threshold", 68 ) 69 parser.addoption( 70 "--restart", 71 default=False, 72 action="store_true", 73 help="Force Unit to restart after every test", 74 ) 75 76 77unit_instance = {} 78_processes = [] 79_fds_info = { 80 'main': {'fds': 0, 'skip': False}, 81 'router': {'name': 'unit: router', 'pid': -1, 'fds': 0, 'skip': False}, 82 'controller': { 83 'name': 'unit: controller', 84 'pid': -1, 85 'fds': 0, 86 'skip': False, 87 }, 88} 89http = TestHTTP() 90 91 92def pytest_configure(config): 93 option.config = config.option 94 95 option.detailed = config.option.detailed 96 option.fds_threshold = config.option.fds_threshold 97 option.print_log = config.option.print_log 98 option.save_log = config.option.save_log 99 option.unsafe = config.option.unsafe 100 option.user = config.option.user 101 option.restart = config.option.restart 102 103 option.generated_tests = {} 104 option.current_dir = os.path.abspath( 105 os.path.join(os.path.dirname(__file__), os.pardir) 106 ) 107 option.test_dir = option.current_dir + '/test' 108 option.architecture = platform.architecture()[0] 109 option.system = platform.system() 110 111 option.cache_dir = tempfile.mkdtemp(prefix='unit-test-cache-') 112 public_dir(option.cache_dir) 113 114 # set stdout to non-blocking 115 116 if option.detailed or option.print_log: 117 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, 0) 118 119 120def print_log_on_assert(func): 121 def inner_function(*args, **kwargs): 122 try: 123 func(*args, **kwargs) 124 except AssertionError as e: 125 _print_log(kwargs.get('log', None)) 126 raise e 127 128 return inner_function 129 130 131def pytest_generate_tests(metafunc): 132 cls = metafunc.cls 133 if ( 134 not hasattr(cls, 'application_type') 135 or cls.application_type == None 136 or cls.application_type == 'external' 137 ): 138 return 139 140 type = cls.application_type 141 142 def generate_tests(versions): 143 metafunc.fixturenames.append('tmp_ct') 144 metafunc.parametrize('tmp_ct', versions) 145 146 for version in versions: 147 option.generated_tests[ 148 metafunc.function.__name__ + '[{}]'.format(version) 149 ] = (type + ' ' + version) 150 151 # take available module from option and generate tests for each version 152 153 for module, prereq_version in cls.prerequisites['modules'].items(): 154 if module in option.available['modules']: 155 available_versions = option.available['modules'][module] 156 157 if prereq_version == 'all': 158 generate_tests(available_versions) 159 160 elif prereq_version == 'any': 161 option.generated_tests[metafunc.function.__name__] = ( 162 type + ' ' + available_versions[0] 163 ) 164 elif callable(prereq_version): 165 generate_tests(list(filter(prereq_version, available_versions))) 166 167 else: 168 raise ValueError( 169 """ 170Unexpected prerequisite version "%s" for module "%s" in %s. 171'all', 'any' or callable expected.""" 172 % (str(prereq_version), module, str(cls)) 173 ) 174 175 176def pytest_sessionstart(session): 177 option.available = {'modules': {}, 'features': {}} 178 179 unit = unit_run() 180 output_version = subprocess.check_output( 181 [unit['unitd'], '--version'], stderr=subprocess.STDOUT 182 ).decode() 183 184 # read unit.log 185 186 for i in range(50): 187 with open(Log.get_path(), 'r') as f: 188 log = f.read() 189 m = re.search('controller started', log) 190 191 if m is None: 192 time.sleep(0.1) 193 else: 194 break 195 196 if m is None: 197 _print_log(log) 198 exit("Unit is writing log too long") 199 200 # discover available modules from unit.log 201 202 for module in re.findall(r'module: ([a-zA-Z]+) (.*) ".*"$', log, re.M): 203 versions = option.available['modules'].setdefault(module[0], []) 204 if module[1] not in versions: 205 versions.append(module[1]) 206 207 # discover modules from check 208 209 option.available['modules']['go'] = check_go() 210 option.available['modules']['njs'] = check_njs(output_version) 211 option.available['modules']['node'] = check_node(option.current_dir) 212 option.available['modules']['openssl'] = check_openssl(output_version) 213 option.available['modules']['regex'] = check_regex(output_version) 214 215 # remove None values 216 217 option.available['modules'] = { 218 k: v for k, v in option.available['modules'].items() if v is not None 219 } 220 221 check_chroot() 222 check_isolation() 223 check_unix_abstract() 224 225 _clear_conf(unit['temp_dir'] + '/control.unit.sock') 226 227 unit_stop() 228 229 _check_alerts() 230 231 if option.restart: 232 shutil.rmtree(unit_instance['temp_dir']) 233 234 235@pytest.hookimpl(tryfirst=True, hookwrapper=True) 236def pytest_runtest_makereport(item, call): 237 # execute all other hooks to obtain the report object 238 outcome = yield 239 rep = outcome.get_result() 240 241 # set a report attribute for each phase of a call, which can 242 # be "setup", "call", "teardown" 243 244 setattr(item, "rep_" + rep.when, rep) 245 246 247@pytest.fixture(scope='class', autouse=True) 248def check_prerequisites(request): 249 cls = request.cls 250 missed = [] 251 252 # check modules 253 254 if 'modules' in cls.prerequisites: 255 available_modules = list(option.available['modules'].keys()) 256 257 for module in cls.prerequisites['modules']: 258 if module in available_modules: 259 continue 260 261 missed.append(module) 262 263 if missed: 264 pytest.skip('Unit has no ' + ', '.join(missed) + ' module(s)') 265 266 # check features 267 268 if 'features' in cls.prerequisites: 269 available_features = list(option.available['features'].keys()) 270 271 for feature in cls.prerequisites['features']: 272 if feature in available_features: 273 continue 274 275 missed.append(feature) 276 277 if missed: 278 pytest.skip(', '.join(missed) + ' feature(s) not supported') 279 280 281@pytest.fixture(autouse=True) 282def run(request): 283 unit = unit_run() 284 285 option.skip_alerts = [ 286 r'read signalfd\(4\) failed', 287 r'sendmsg.+failed', 288 r'recvmsg.+failed', 289 ] 290 option.skip_sanitizer = False 291 292 _fds_info['main']['skip'] = False 293 _fds_info['router']['skip'] = False 294 _fds_info['controller']['skip'] = False 295 296 yield 297 298 # stop unit 299 300 error_stop_unit = unit_stop() 301 error_stop_processes = stop_processes() 302 303 # prepare log 304 305 with Log.open(encoding='utf-8') as f: 306 log = f.read() 307 Log.set_pos(f.tell()) 308 309 if not option.save_log and option.restart: 310 shutil.rmtree(unit['temp_dir']) 311 Log.set_pos(0) 312 313 # clean temp_dir before the next test 314 315 if not option.restart: 316 _clear_conf(unit['temp_dir'] + '/control.unit.sock', log=log) 317 318 for item in os.listdir(unit['temp_dir']): 319 if item not in [ 320 'control.unit.sock', 321 'state', 322 'unit.pid', 323 'unit.log', 324 ]: 325 path = os.path.join(unit['temp_dir'], item) 326 327 public_dir(path) 328 329 if os.path.isfile(path) or stat.S_ISSOCK(os.stat(path).st_mode): 330 os.remove(path) 331 else: 332 for attempt in range(10): 333 try: 334 shutil.rmtree(path) 335 break 336 except OSError as err: 337 if err.errno != 16: 338 raise 339 time.sleep(1) 340 341 # check descriptors 342 343 _check_fds(log=log) 344 345 # check processes id's and amount 346 347 _check_processes() 348 349 # print unit.log in case of error 350 351 if hasattr(request.node, 'rep_call') and request.node.rep_call.failed: 352 _print_log(log) 353 354 if error_stop_unit or error_stop_processes: 355 _print_log(log) 356 357 # check unit.log for errors 358 359 assert error_stop_unit is None, 'stop unit' 360 assert error_stop_processes is None, 'stop processes' 361 362 _check_alerts(log=log) 363 364 365def unit_run(state_dir=None): 366 global unit_instance 367 368 if not option.restart and 'unitd' in unit_instance: 369 return unit_instance 370 371 build_dir = option.current_dir + '/build' 372 unitd = build_dir + '/unitd' 373 374 if not os.path.isfile(unitd): 375 exit('Could not find unit') 376 377 temp_dir = tempfile.mkdtemp(prefix='unit-test-') 378 public_dir(temp_dir) 379 380 if oct(stat.S_IMODE(os.stat(build_dir).st_mode)) != '0o777': 381 public_dir(build_dir) 382 383 state = temp_dir + '/state' if state_dir is None else state_dir 384 if not os.path.isdir(state): 385 os.mkdir(state) 386 387 unitd_args = [ 388 unitd, 389 '--no-daemon', 390 '--modules', 391 build_dir, 392 '--state', 393 state, 394 '--pid', 395 temp_dir + '/unit.pid', 396 '--log', 397 temp_dir + '/unit.log', 398 '--control', 399 'unix:' + temp_dir + '/control.unit.sock', 400 '--tmp', 401 temp_dir, 402 ] 403 404 if option.user: 405 unitd_args.extend(['--user', option.user]) 406 407 with open(temp_dir + '/unit.log', 'w') as log: 408 unit_instance['process'] = subprocess.Popen(unitd_args, stderr=log) 409 410 Log.temp_dir = temp_dir 411 412 if not waitforfiles(temp_dir + '/control.unit.sock'): 413 _print_log() 414 exit('Could not start unit') 415 416 unit_instance['temp_dir'] = temp_dir 417 unit_instance['control_sock'] = temp_dir + '/control.unit.sock' 418 unit_instance['unitd'] = unitd 419 420 option.temp_dir = temp_dir 421 422 with open(temp_dir + '/unit.pid', 'r') as f: 423 unit_instance['pid'] = f.read().rstrip() 424 425 if state_dir is None: 426 _clear_conf(unit_instance['temp_dir'] + '/control.unit.sock') 427 428 _fds_info['main']['fds'] = _count_fds(unit_instance['pid']) 429 430 router = _fds_info['router'] 431 router['pid'] = pid_by_name(router['name']) 432 router['fds'] = _count_fds(router['pid']) 433 434 controller = _fds_info['controller'] 435 controller['pid'] = pid_by_name(controller['name']) 436 controller['fds'] = _count_fds(controller['pid']) 437 438 Status._check_zeros() 439 440 return unit_instance 441 442 443def unit_stop(): 444 if not option.restart: 445 if inspect.stack()[1].function.startswith('test_'): 446 pytest.skip('no restart mode') 447 448 return 449 450 # check zombies 451 452 out = subprocess.check_output( 453 ['ps', 'ax', '-o', 'state', '-o', 'ppid'] 454 ).decode() 455 z_ppids = re.findall(r'Z\s*(\d+)', out) 456 assert unit_instance['pid'] not in z_ppids, 'no zombies' 457 458 # terminate unit 459 460 p = unit_instance['process'] 461 462 if p.poll() is not None: 463 return 464 465 p.send_signal(signal.SIGQUIT) 466 467 try: 468 retcode = p.wait(15) 469 if retcode: 470 return 'Child process terminated with code ' + str(retcode) 471 472 except KeyboardInterrupt: 473 p.kill() 474 raise 475 476 except: 477 p.kill() 478 return 'Could not terminate unit' 479 480 481@print_log_on_assert 482def _check_alerts(*, log=None): 483 if log is None: 484 with Log.open(encoding='utf-8') as f: 485 log = f.read() 486 487 found = False 488 489 alerts = re.findall(r'.+\[alert\].+', log) 490 491 if alerts: 492 print('\nAll alerts/sanitizer errors found in log:') 493 [print(alert) for alert in alerts] 494 found = True 495 496 if option.skip_alerts: 497 for skip in option.skip_alerts: 498 alerts = [al for al in alerts if re.search(skip, al) is None] 499 500 assert not alerts, 'alert(s)' 501 502 if not option.skip_sanitizer: 503 sanitizer_errors = re.findall('.+Sanitizer.+', log) 504 505 assert not sanitizer_errors, 'sanitizer error(s)' 506 507 if found: 508 print('skipped.') 509 510 511def _print_log(log=None): 512 path = Log.get_path() 513 514 print('Path to unit.log:\n' + path + '\n') 515 516 if option.print_log: 517 os.set_blocking(sys.stdout.fileno(), True) 518 sys.stdout.flush() 519 520 if log is None: 521 with open(path, 'r', encoding='utf-8', errors='ignore') as f: 522 shutil.copyfileobj(f, sys.stdout) 523 else: 524 sys.stdout.write(log) 525 526 527@print_log_on_assert 528def _clear_conf(sock, *, log=None): 529 resp = http.put( 530 url='/config', 531 sock_type='unix', 532 addr=sock, 533 body=json.dumps({"listeners": {}, "applications": {}}), 534 )['body'] 535 536 assert 'success' in resp, 'clear conf' 537 538 if 'openssl' not in option.available['modules']: 539 return 540 541 try: 542 certs = json.loads( 543 http.get(url='/certificates', sock_type='unix', addr=sock)['body'] 544 ).keys() 545 546 except json.JSONDecodeError: 547 pytest.fail('Can\'t parse certificates list.') 548 549 for cert in certs: 550 resp = http.delete( 551 url='/certificates/' + cert, 552 sock_type='unix', 553 addr=sock, 554 )['body'] 555 556 assert 'success' in resp, 'remove certificate' 557 558 559def _check_processes(): 560 router_pid = _fds_info['router']['pid'] 561 controller_pid = _fds_info['controller']['pid'] 562 unit_pid = unit_instance['pid'] 563 564 for i in range(600): 565 out = ( 566 subprocess.check_output( 567 ['ps', '-ax', '-o', 'pid', '-o', 'ppid', '-o', 'command'] 568 ) 569 .decode() 570 .splitlines() 571 ) 572 out = [l for l in out if unit_pid in l] 573 574 if len(out) <= 3: 575 break 576 577 time.sleep(0.1) 578 579 if option.restart: 580 assert len(out) == 0, 'all termimated' 581 return 582 583 assert len(out) == 3, 'main, router, and controller expected' 584 585 out = [l for l in out if 'unit: main' not in l] 586 assert len(out) == 2, 'one main' 587 588 out = [ 589 l 590 for l in out 591 if re.search(router_pid + r'\s+' + unit_pid + r'.*unit: router', l) 592 is None 593 ] 594 assert len(out) == 1, 'one router' 595 596 out = [ 597 l 598 for l in out 599 if re.search( 600 controller_pid + r'\s+' + unit_pid + r'.*unit: controller', l 601 ) 602 is None 603 ] 604 assert len(out) == 0, 'one controller' 605 606 607@print_log_on_assert 608def _check_fds(*, log=None): 609 def waitforfds(diff): 610 for i in range(600): 611 fds_diff = diff() 612 613 if fds_diff <= option.fds_threshold: 614 break 615 616 time.sleep(0.1) 617 618 return fds_diff 619 620 ps = _fds_info['main'] 621 if not ps['skip']: 622 fds_diff = waitforfds( 623 lambda: _count_fds(unit_instance['pid']) - ps['fds'] 624 ) 625 ps['fds'] += fds_diff 626 627 assert fds_diff <= option.fds_threshold, 'descriptors leak main process' 628 629 else: 630 ps['fds'] = _count_fds(unit_instance['pid']) 631 632 for name in ['controller', 'router']: 633 ps = _fds_info[name] 634 ps_pid = ps['pid'] 635 ps['pid'] = pid_by_name(ps['name']) 636 637 if not ps['skip']: 638 fds_diff = waitforfds(lambda: _count_fds(ps['pid']) - ps['fds']) 639 ps['fds'] += fds_diff 640 641 if not option.restart: 642 assert ps['pid'] == ps_pid, 'same pid %s' % name 643 644 assert fds_diff <= option.fds_threshold, ( 645 'descriptors leak %s' % name 646 ) 647 648 else: 649 ps['fds'] = _count_fds(ps['pid']) 650 651 652def _count_fds(pid): 653 procfile = '/proc/%s/fd' % pid 654 if os.path.isdir(procfile): 655 return len(os.listdir(procfile)) 656 657 try: 658 out = subprocess.check_output( 659 ['procstat', '-f', pid], 660 stderr=subprocess.STDOUT, 661 ).decode() 662 return len(out.splitlines()) 663 664 except (FileNotFoundError, TypeError, subprocess.CalledProcessError): 665 pass 666 667 try: 668 out = subprocess.check_output( 669 ['lsof', '-n', '-p', pid], 670 stderr=subprocess.STDOUT, 671 ).decode() 672 return len(out.splitlines()) 673 674 except (FileNotFoundError, TypeError, subprocess.CalledProcessError): 675 pass 676 677 return 0 678 679 680def run_process(target, *args): 681 global _processes 682 683 process = Process(target=target, args=args) 684 process.start() 685 686 _processes.append(process) 687 688 689def stop_processes(): 690 if not _processes: 691 return 692 693 fail = False 694 for process in _processes: 695 if process.is_alive(): 696 process.terminate() 697 process.join(timeout=15) 698 699 if process.is_alive(): 700 fail = True 701 702 if fail: 703 return 'Fail to stop process(es)' 704 705 706def pid_by_name(name): 707 output = subprocess.check_output(['ps', 'ax', '-O', 'ppid']).decode() 708 m = re.search( 709 r'\s*(\d+)\s*' + str(unit_instance['pid']) + r'.*' + name, output 710 ) 711 return None if m is None else m.group(1) 712 713 714def find_proc(name, ps_output): 715 return re.findall(str(unit_instance['pid']) + r'.*' + name, ps_output) 716 717 718@pytest.fixture() 719def skip_alert(): 720 def _skip(*alerts): 721 option.skip_alerts.extend(alerts) 722 723 return _skip 724 725 726@pytest.fixture() 727def skip_fds_check(): 728 def _skip(main=False, router=False, controller=False): 729 _fds_info['main']['skip'] = main 730 _fds_info['router']['skip'] = router 731 _fds_info['controller']['skip'] = controller 732 733 return _skip 734 735 736@pytest.fixture 737def temp_dir(request): 738 return unit_instance['temp_dir'] 739 740 741@pytest.fixture 742def is_unsafe(request): 743 return request.config.getoption("--unsafe") 744 745 746@pytest.fixture 747def is_su(request): 748 return os.geteuid() == 0 749 750 751@pytest.fixture 752def unit_pid(request): 753 return unit_instance['process'].pid 754 755 756def pytest_sessionfinish(session): 757 if not option.restart and option.save_log: 758 print('Path to unit.log:\n' + Log.get_path() + '\n') 759 760 option.restart = True 761 762 unit_stop() 763 764 public_dir(option.cache_dir) 765 shutil.rmtree(option.cache_dir) 766 767 if not option.save_log and os.path.isdir(option.temp_dir): 768 public_dir(option.temp_dir) 769 shutil.rmtree(option.temp_dir) 770