xref: /unit/test/test_python_application.py (revision 2086:3baca957c0dd)
1import grp
2import os
3import pwd
4import re
5import time
6
7import pytest
8from unit.applications.lang.python import TestApplicationPython
9
10
11class TestPythonApplication(TestApplicationPython):
12    prerequisites = {'modules': {'python': 'all'}}
13
14    def test_python_application_variables(self):
15        self.load('variables')
16
17        body = 'Test body string.'
18
19        resp = self.http(
20            b"""POST / HTTP/1.1
21Host: localhost
22Content-Length: %d
23Custom-Header: blah
24Custom-hEader: Blah
25Content-Type: text/html
26Connection: close
27custom-header: BLAH
28
29%s"""
30            % (len(body), body.encode()),
31            raw=True,
32        )
33
34        assert resp['status'] == 200, 'status'
35        headers = resp['headers']
36        header_server = headers.pop('Server')
37        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
38        assert (
39            headers.pop('Server-Software') == header_server
40        ), 'server software header'
41
42        date = headers.pop('Date')
43        assert date[-4:] == ' GMT', 'date header timezone'
44        assert (
45            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
46        ), 'date header'
47
48        assert headers == {
49            'Connection': 'close',
50            'Content-Length': str(len(body)),
51            'Content-Type': 'text/html',
52            'Request-Method': 'POST',
53            'Request-Uri': '/',
54            'Http-Host': 'localhost',
55            'Server-Protocol': 'HTTP/1.1',
56            'Custom-Header': 'blah, Blah, BLAH',
57            'Wsgi-Version': '(1, 0)',
58            'Wsgi-Url-Scheme': 'http',
59            'Wsgi-Multithread': 'False',
60            'Wsgi-Multiprocess': 'True',
61            'Wsgi-Run-Once': 'False',
62        }, 'headers'
63        assert resp['body'] == body, 'body'
64
65    def test_python_application_query_string(self):
66        self.load('query_string')
67
68        resp = self.get(url='/?var1=val1&var2=val2')
69
70        assert (
71            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
72        ), 'Query-String header'
73
74    def test_python_application_query_string_space(self):
75        self.load('query_string')
76
77        resp = self.get(url='/ ?var1=val1&var2=val2')
78        assert (
79            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
80        ), 'Query-String space'
81
82        resp = self.get(url='/ %20?var1=val1&var2=val2')
83        assert (
84            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
85        ), 'Query-String space 2'
86
87        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
88        assert (
89            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
90        ), 'Query-String space 3'
91
92        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
93        assert (
94            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
95        ), 'Query-String space 4'
96
97    def test_python_application_query_string_empty(self):
98        self.load('query_string')
99
100        resp = self.get(url='/?')
101
102        assert resp['status'] == 200, 'query string empty status'
103        assert resp['headers']['Query-String'] == '', 'query string empty'
104
105    def test_python_application_query_string_absent(self):
106        self.load('query_string')
107
108        resp = self.get()
109
110        assert resp['status'] == 200, 'query string absent status'
111        assert resp['headers']['Query-String'] == '', 'query string absent'
112
113    @pytest.mark.skip('not yet')
114    def test_python_application_server_port(self):
115        self.load('server_port')
116
117        assert (
118            self.get()['headers']['Server-Port'] == '7080'
119        ), 'Server-Port header'
120
121    @pytest.mark.skip('not yet')
122    def test_python_application_working_directory_invalid(self):
123        self.load('empty')
124
125        assert 'success' in self.conf(
126            '"/blah"', 'applications/empty/working_directory'
127        ), 'configure invalid working_directory'
128
129        assert self.get()['status'] == 500, 'status'
130
131    def test_python_application_204_transfer_encoding(self):
132        self.load('204_no_content')
133
134        assert (
135            'Transfer-Encoding' not in self.get()['headers']
136        ), '204 header transfer encoding'
137
138    def test_python_application_ctx_iter_atexit(self):
139        self.load('ctx_iter_atexit')
140
141        resp = self.post(
142            headers={
143                'Host': 'localhost',
144                'Connection': 'close',
145                'Content-Type': 'text/html',
146            },
147            body='0123456789',
148        )
149
150        assert resp['status'] == 200, 'ctx iter status'
151        assert resp['body'] == '0123456789', 'ctx iter body'
152
153        assert 'success' in self.conf({"listeners": {}, "applications": {}})
154
155        assert (
156            self.wait_for_record(r'RuntimeError') is not None
157        ), 'ctx iter atexit'
158
159    def test_python_keepalive_body(self):
160        self.load('mirror')
161
162        assert self.get()['status'] == 200, 'init'
163
164        body = '0123456789' * 500
165        (resp, sock) = self.post(
166            headers={
167                'Host': 'localhost',
168                'Connection': 'keep-alive',
169                'Content-Type': 'text/html',
170            },
171            start=True,
172            body=body,
173            read_timeout=1,
174        )
175
176        assert resp['body'] == body, 'keep-alive 1'
177
178        body = '0123456789'
179        resp = self.post(
180            headers={
181                'Host': 'localhost',
182                'Connection': 'close',
183                'Content-Type': 'text/html',
184            },
185            sock=sock,
186            body=body,
187        )
188
189        assert resp['body'] == body, 'keep-alive 2'
190
191    def test_python_keepalive_reconfigure(self):
192        self.load('mirror')
193
194        assert self.get()['status'] == 200, 'init'
195
196        body = '0123456789'
197        conns = 3
198        socks = []
199
200        for i in range(conns):
201            (resp, sock) = self.post(
202                headers={
203                    'Host': 'localhost',
204                    'Connection': 'keep-alive',
205                    'Content-Type': 'text/html',
206                },
207                start=True,
208                body=body,
209                read_timeout=1,
210            )
211
212            assert resp['body'] == body, 'keep-alive open'
213
214            self.load('mirror', processes=i + 1)
215
216            socks.append(sock)
217
218        for i in range(conns):
219            (resp, sock) = self.post(
220                headers={
221                    'Host': 'localhost',
222                    'Connection': 'keep-alive',
223                    'Content-Type': 'text/html',
224                },
225                start=True,
226                sock=socks[i],
227                body=body,
228                read_timeout=1,
229            )
230
231            assert resp['body'] == body, 'keep-alive request'
232
233            self.load('mirror', processes=i + 1)
234
235        for i in range(conns):
236            resp = self.post(
237                headers={
238                    'Host': 'localhost',
239                    'Connection': 'close',
240                    'Content-Type': 'text/html',
241                },
242                sock=socks[i],
243                body=body,
244            )
245
246            assert resp['body'] == body, 'keep-alive close'
247
248            self.load('mirror', processes=i + 1)
249
250    def test_python_keepalive_reconfigure_2(self):
251        self.load('mirror')
252
253        assert self.get()['status'] == 200, 'init'
254
255        body = '0123456789'
256
257        (resp, sock) = self.post(
258            headers={
259                'Host': 'localhost',
260                'Connection': 'keep-alive',
261                'Content-Type': 'text/html',
262            },
263            start=True,
264            body=body,
265            read_timeout=1,
266        )
267
268        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
269
270        self.load('empty')
271
272        assert self.get()['status'] == 200, 'init'
273
274        (resp, sock) = self.post(
275            headers={
276                'Host': 'localhost',
277                'Connection': 'close',
278                'Content-Type': 'text/html',
279            },
280            start=True,
281            sock=sock,
282            body=body,
283        )
284
285        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
286        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
287
288        assert 'success' in self.conf(
289            {"listeners": {}, "applications": {}}
290        ), 'reconfigure 2 clear configuration'
291
292        resp = self.get(sock=sock)
293
294        assert resp == {}, 'reconfigure 2 keep-alive 3'
295
296    def test_python_atexit(self):
297        self.load('atexit')
298
299        self.get()
300
301        assert 'success' in self.conf({"listeners": {}, "applications": {}})
302
303        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
304
305    def test_python_process_switch(self):
306        self.load('delayed', processes=2)
307
308        self.get(
309            headers={
310                'Host': 'localhost',
311                'Content-Length': '0',
312                'X-Delay': '5',
313                'Connection': 'close',
314            },
315            no_recv=True,
316        )
317
318        headers_delay_1 = {
319            'Connection': 'close',
320            'Host': 'localhost',
321            'Content-Length': '0',
322            'X-Delay': '1',
323        }
324
325        self.get(headers=headers_delay_1, no_recv=True)
326
327        time.sleep(0.5)
328
329        for _ in range(10):
330            self.get(headers=headers_delay_1, no_recv=True)
331
332        self.get(headers=headers_delay_1)
333
334    @pytest.mark.skip('not yet')
335    def test_python_application_start_response_exit(self):
336        self.load('start_response_exit')
337
338        assert self.get()['status'] == 500, 'start response exit'
339
340    def test_python_application_input_iter(self):
341        self.load('input_iter')
342
343        body = '''0123456789
344next line
345
346last line'''
347
348        resp = self.post(body=body)
349        assert resp['body'] == body, 'input iter'
350        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
351
352    def test_python_application_input_readline(self):
353        self.load('input_readline')
354
355        body = '''0123456789
356next line
357
358last line'''
359
360        resp = self.post(body=body)
361        assert resp['body'] == body, 'input readline'
362        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
363
364    def test_python_application_input_readline_size(self):
365        self.load('input_readline_size')
366
367        body = '''0123456789
368next line
369
370last line'''
371
372        assert self.post(body=body)['body'] == body, 'input readline size'
373        assert (
374            self.post(body='0123')['body'] == '0123'
375        ), 'input readline size less'
376
377    def test_python_application_input_readlines(self):
378        self.load('input_readlines')
379
380        body = '''0123456789
381next line
382
383last line'''
384
385        resp = self.post(body=body)
386        assert resp['body'] == body, 'input readlines'
387        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
388
389    def test_python_application_input_readlines_huge(self):
390        self.load('input_readlines')
391
392        body = (
393            '''0123456789 abcdefghi
394next line: 0123456789 abcdefghi
395
396last line: 987654321
397'''
398            * 512
399        )
400
401        assert (
402            self.post(body=body, read_buffer_size=16384)['body'] == body
403        ), 'input readlines huge'
404
405    def test_python_application_input_read_length(self):
406        self.load('input_read_length')
407
408        body = '0123456789'
409
410        resp = self.post(
411            headers={
412                'Host': 'localhost',
413                'Input-Length': '5',
414                'Connection': 'close',
415            },
416            body=body,
417        )
418
419        assert resp['body'] == body[:5], 'input read length lt body'
420
421        resp = self.post(
422            headers={
423                'Host': 'localhost',
424                'Input-Length': '15',
425                'Connection': 'close',
426            },
427            body=body,
428        )
429
430        assert resp['body'] == body, 'input read length gt body'
431
432        resp = self.post(
433            headers={
434                'Host': 'localhost',
435                'Input-Length': '0',
436                'Connection': 'close',
437            },
438            body=body,
439        )
440
441        assert resp['body'] == '', 'input read length zero'
442
443        resp = self.post(
444            headers={
445                'Host': 'localhost',
446                'Input-Length': '-1',
447                'Connection': 'close',
448            },
449            body=body,
450        )
451
452        assert resp['body'] == body, 'input read length negative'
453
454    @pytest.mark.skip('not yet')
455    def test_python_application_errors_write(self):
456        self.load('errors_write')
457
458        self.get()
459
460        assert (
461            self.wait_for_record(r'\[error\].+Error in application\.')
462            is not None
463        ), 'errors write'
464
465    def test_python_application_body_array(self):
466        self.load('body_array')
467
468        assert self.get()['body'] == '0123456789', 'body array'
469
470    def test_python_application_body_io(self):
471        self.load('body_io')
472
473        assert self.get()['body'] == '0123456789', 'body io'
474
475    def test_python_application_body_io_file(self):
476        self.load('body_io_file')
477
478        assert self.get()['body'] == 'body\n', 'body io file'
479
480    @pytest.mark.skip('not yet')
481    def test_python_application_syntax_error(self, skip_alert):
482        skip_alert(r'Python failed to import module "wsgi"')
483        self.load('syntax_error')
484
485        assert self.get()['status'] == 500, 'syntax error'
486
487    def test_python_application_loading_error(self, skip_alert):
488        skip_alert(r'Python failed to import module "blah"')
489
490        self.load('empty', module="blah")
491
492        assert self.get()['status'] == 503, 'loading error'
493
494    def test_python_application_close(self):
495        self.load('close')
496
497        self.get()
498
499        assert self.wait_for_record(r'Close called\.') is not None, 'close'
500
501    def test_python_application_close_error(self):
502        self.load('close_error')
503
504        self.get()
505
506        assert (
507            self.wait_for_record(r'Close called\.') is not None
508        ), 'close error'
509
510    def test_python_application_not_iterable(self):
511        self.load('not_iterable')
512
513        self.get()
514
515        assert (
516            self.wait_for_record(
517                r'\[error\].+the application returned not an iterable object'
518            )
519            is not None
520        ), 'not iterable'
521
522    def test_python_application_write(self):
523        self.load('write')
524
525        assert self.get()['body'] == '0123456789', 'write'
526
527    def test_python_application_threading(self):
528        """wait_for_record() timeouts after 5s while every thread works at
529        least 3s.  So without releasing GIL test should fail.
530        """
531
532        self.load('threading')
533
534        for _ in range(10):
535            self.get(no_recv=True)
536
537        assert (
538            self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None
539        ), 'last thread finished'
540
541    def test_python_application_iter_exception(self):
542        self.load('iter_exception')
543
544        # Default request doesn't lead to the exception.
545
546        resp = self.get(
547            headers={
548                'Host': 'localhost',
549                'X-Skip': '9',
550                'X-Chunked': '1',
551                'Connection': 'close',
552            }
553        )
554        assert resp['status'] == 200, 'status'
555        assert resp['body'] == 'XXXXXXX', 'body'
556
557        # Exception before start_response().
558
559        assert self.get()['status'] == 503, 'error'
560
561        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
562        assert (
563            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
564            is not None
565        ), 'first exception raise'
566        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
567
568        # Exception after start_response(), before first write().
569
570        assert (
571            self.get(
572                headers={
573                    'Host': 'localhost',
574                    'X-Skip': '1',
575                    'Connection': 'close',
576                }
577            )['status']
578            == 503
579        ), 'error 2'
580
581        assert (
582            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
583            is not None
584        ), 'exception raise second'
585        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
586
587        # Exception after first write(), before first __next__().
588
589        _, sock = self.get(
590            headers={
591                'Host': 'localhost',
592                'X-Skip': '2',
593                'Connection': 'keep-alive',
594            },
595            start=True,
596        )
597
598        assert (
599            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
600            is not None
601        ), 'exception raise third'
602        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
603
604        assert self.get(sock=sock) == {}, 'closed connection'
605
606        # Exception after first write(), before first __next__(),
607        # chunked (incomplete body).
608
609        resp = self.get(
610            headers={
611                'Host': 'localhost',
612                'X-Skip': '2',
613                'X-Chunked': '1',
614                'Connection': 'close',
615            },
616            raw_resp=True,
617        )
618        if resp:
619            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
620        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
621
622        # Exception in __next__().
623
624        _, sock = self.get(
625            headers={
626                'Host': 'localhost',
627                'X-Skip': '3',
628                'Connection': 'keep-alive',
629            },
630            start=True,
631        )
632
633        assert (
634            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
635            is not None
636        ), 'exception raise next'
637        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
638
639        assert self.get(sock=sock) == {}, 'closed connection 2'
640
641        # Exception in __next__(), chunked (incomplete body).
642
643        resp = self.get(
644            headers={
645                'Host': 'localhost',
646                'X-Skip': '3',
647                'X-Chunked': '1',
648                'Connection': 'close',
649            },
650            raw_resp=True,
651        )
652        if resp:
653            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
654        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
655
656        # Exception before start_response() and in close().
657
658        assert (
659            self.get(
660                headers={
661                    'Host': 'localhost',
662                    'X-Not-Skip-Close': '1',
663                    'Connection': 'close',
664                }
665            )['status']
666            == 503
667        ), 'error'
668
669        assert (
670            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
671            is not None
672        ), 'exception raise close'
673        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
674
675    def test_python_user_group(self, is_su):
676        if not is_su:
677            pytest.skip('requires root')
678
679        nobody_uid = pwd.getpwnam('nobody').pw_uid
680
681        group = 'nobody'
682
683        try:
684            group_id = grp.getgrnam(group).gr_gid
685        except KeyError:
686            group = 'nogroup'
687            group_id = grp.getgrnam(group).gr_gid
688
689        self.load('user_group')
690
691        obj = self.getjson()['body']
692        assert obj['UID'] == nobody_uid, 'nobody uid'
693        assert obj['GID'] == group_id, 'nobody gid'
694
695        self.load('user_group', user='nobody')
696
697        obj = self.getjson()['body']
698        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
699        assert obj['GID'] == group_id, 'nobody gid user=nobody'
700
701        self.load('user_group', user='nobody', group=group)
702
703        obj = self.getjson()['body']
704        assert obj['UID'] == nobody_uid, (
705            'nobody uid user=nobody group=%s' % group
706        )
707
708        assert obj['GID'] == group_id, 'nobody gid user=nobody group=%s' % group
709
710        self.load('user_group', group=group)
711
712        obj = self.getjson()['body']
713        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
714
715        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
716
717        self.load('user_group', user='root')
718
719        obj = self.getjson()['body']
720        assert obj['UID'] == 0, 'root uid user=root'
721        assert obj['GID'] == 0, 'root gid user=root'
722
723        group = 'root'
724
725        try:
726            grp.getgrnam(group)
727            group = True
728        except KeyError:
729            group = False
730
731        if group:
732            self.load('user_group', user='root', group='root')
733
734            obj = self.getjson()['body']
735            assert obj['UID'] == 0, 'root uid user=root group=root'
736            assert obj['GID'] == 0, 'root gid user=root group=root'
737
738            self.load('user_group', group='root')
739
740            obj = self.getjson()['body']
741            assert obj['UID'] == nobody_uid, 'root uid group=root'
742            assert obj['GID'] == 0, 'root gid group=root'
743
744    def test_python_application_callable(self, skip_alert):
745        skip_alert(r'Python failed to get "blah" from module')
746        self.load('callable')
747
748        assert self.get()['status'] == 204, 'default application response'
749
750        self.load('callable', callable="app")
751
752        assert self.get()['status'] == 200, 'callable response'
753
754        self.load('callable', callable="blah")
755
756        assert self.get()['status'] not in [200, 204], 'callable response inv'
757
758    def test_python_application_path(self):
759        self.load('path')
760
761        def set_path(path):
762            assert 'success' in self.conf(path, 'applications/path/path')
763
764        def get_path():
765            return self.get()['body'].split(os.pathsep)
766
767        default_path = self.conf_get('/config/applications/path/path')
768        assert 'success' in self.conf(
769            {"PYTHONPATH": default_path},
770            '/config/applications/path/environment',
771        )
772
773        self.conf_delete('/config/applications/path/path')
774        sys_path = get_path()
775
776        set_path('"/blah"')
777        assert ['/blah', *sys_path] == get_path(), 'check path'
778
779        set_path('"/new"')
780        assert ['/new', *sys_path] == get_path(), 'check path update'
781
782        set_path('["/blah1", "/blah2"]')
783        assert [
784            '/blah1',
785            '/blah2',
786            *sys_path,
787        ] == get_path(), 'check path array'
788
789    def test_python_application_path_invalid(self):
790        self.load('path')
791
792        def check_path(path):
793            assert 'error' in self.conf(path, 'applications/path/path')
794
795        check_path('{}')
796        check_path('["/blah", []]')
797
798    def test_python_application_threads(self):
799        self.load('threads', threads=4)
800
801        socks = []
802
803        for i in range(4):
804            (_, sock) = self.get(
805                headers={
806                    'Host': 'localhost',
807                    'X-Delay': '2',
808                    'Connection': 'close',
809                },
810                no_recv=True,
811                start=True,
812            )
813
814            socks.append(sock)
815
816        threads = set()
817
818        for sock in socks:
819            resp = self.recvall(sock).decode('utf-8')
820
821            self.log_in(resp)
822
823            resp = self._resp_to_dict(resp)
824
825            assert resp['status'] == 200, 'status'
826
827            threads.add(resp['headers']['X-Thread'])
828
829            assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread'
830
831            sock.close()
832
833        assert len(socks) == len(threads), 'threads differs'
834