xref: /unit/test/test_python_application.py (revision 2212:975ab1624784)
1import grp
2import os
3import pwd
4import re
5import time
6
7import pytest
8from unit.applications.lang.python import TestApplicationPython
9
10
11class TestPythonApplication(TestApplicationPython):
12    prerequisites = {'modules': {'python': 'all'}}
13
14    def test_python_application_variables(self):
15        self.load('variables')
16
17        body = 'Test body string.'
18
19        resp = self.http(
20            b"""POST / HTTP/1.1
21Host: localhost
22Content-Length: %d
23Custom-Header: blah
24Custom-hEader: Blah
25Content-Type: text/html
26Connection: close
27custom-header: BLAH
28
29%s"""
30            % (len(body), body.encode()),
31            raw=True,
32        )
33
34        assert resp['status'] == 200, 'status'
35        headers = resp['headers']
36        header_server = headers.pop('Server')
37        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
38        assert (
39            headers.pop('Server-Software') == header_server
40        ), 'server software header'
41
42        date = headers.pop('Date')
43        assert date[-4:] == ' GMT', 'date header timezone'
44        assert (
45            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
46        ), 'date header'
47
48        assert headers == {
49            'Connection': 'close',
50            'Content-Length': str(len(body)),
51            'Content-Type': 'text/html',
52            'Request-Method': 'POST',
53            'Request-Uri': '/',
54            'Http-Host': 'localhost',
55            'Server-Protocol': 'HTTP/1.1',
56            'Custom-Header': 'blah, Blah, BLAH',
57            'Wsgi-Version': '(1, 0)',
58            'Wsgi-Url-Scheme': 'http',
59            'Wsgi-Multithread': 'False',
60            'Wsgi-Multiprocess': 'True',
61            'Wsgi-Run-Once': 'False',
62        }, 'headers'
63        assert resp['body'] == body, 'body'
64
65    def test_python_application_query_string(self):
66        self.load('query_string')
67
68        resp = self.get(url='/?var1=val1&var2=val2')
69
70        assert (
71            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
72        ), 'Query-String header'
73
74    def test_python_application_query_string_space(self):
75        self.load('query_string')
76
77        resp = self.get(url='/ ?var1=val1&var2=val2')
78        assert (
79            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
80        ), 'Query-String space'
81
82        resp = self.get(url='/ %20?var1=val1&var2=val2')
83        assert (
84            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
85        ), 'Query-String space 2'
86
87        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
88        assert (
89            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
90        ), 'Query-String space 3'
91
92        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
93        assert (
94            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
95        ), 'Query-String space 4'
96
97    def test_python_application_query_string_empty(self):
98        self.load('query_string')
99
100        resp = self.get(url='/?')
101
102        assert resp['status'] == 200, 'query string empty status'
103        assert resp['headers']['Query-String'] == '', 'query string empty'
104
105    def test_python_application_query_string_absent(self):
106        self.load('query_string')
107
108        resp = self.get()
109
110        assert resp['status'] == 200, 'query string absent status'
111        assert resp['headers']['Query-String'] == '', 'query string absent'
112
113    @pytest.mark.skip('not yet')
114    def test_python_application_server_port(self):
115        self.load('server_port')
116
117        assert (
118            self.get()['headers']['Server-Port'] == '7080'
119        ), 'Server-Port header'
120
121    @pytest.mark.skip('not yet')
122    def test_python_application_working_directory_invalid(self):
123        self.load('empty')
124
125        assert 'success' in self.conf(
126            '"/blah"', 'applications/empty/working_directory'
127        ), 'configure invalid working_directory'
128
129        assert self.get()['status'] == 500, 'status'
130
131    def test_python_application_204_transfer_encoding(self):
132        self.load('204_no_content')
133
134        assert (
135            'Transfer-Encoding' not in self.get()['headers']
136        ), '204 header transfer encoding'
137
138    def test_python_application_ctx_iter_atexit(self):
139        self.load('ctx_iter_atexit')
140
141        resp = self.post(body='0123456789')
142
143        assert resp['status'] == 200, 'ctx iter status'
144        assert resp['body'] == '0123456789', 'ctx iter body'
145
146        assert 'success' in self.conf({"listeners": {}, "applications": {}})
147
148        assert (
149            self.wait_for_record(r'RuntimeError') is not None
150        ), 'ctx iter atexit'
151
152    def test_python_keepalive_body(self):
153        self.load('mirror')
154
155        assert self.get()['status'] == 200, 'init'
156
157        body = '0123456789' * 500
158        (resp, sock) = self.post(
159            headers={
160                'Host': 'localhost',
161                'Connection': 'keep-alive',
162            },
163            start=True,
164            body=body,
165            read_timeout=1,
166        )
167
168        assert resp['body'] == body, 'keep-alive 1'
169
170        body = '0123456789'
171        resp = self.post(sock=sock, body=body)
172
173        assert resp['body'] == body, 'keep-alive 2'
174
175    def test_python_keepalive_reconfigure(self):
176        self.load('mirror')
177
178        assert self.get()['status'] == 200, 'init'
179
180        body = '0123456789'
181        conns = 3
182        socks = []
183
184        for i in range(conns):
185            (resp, sock) = self.post(
186                headers={
187                    'Host': 'localhost',
188                    'Connection': 'keep-alive',
189                },
190                start=True,
191                body=body,
192                read_timeout=1,
193            )
194
195            assert resp['body'] == body, 'keep-alive open'
196
197            self.load('mirror', processes=i + 1)
198
199            socks.append(sock)
200
201        for i in range(conns):
202            (resp, sock) = self.post(
203                headers={
204                    'Host': 'localhost',
205                    'Connection': 'keep-alive',
206                },
207                start=True,
208                sock=socks[i],
209                body=body,
210                read_timeout=1,
211            )
212
213            assert resp['body'] == body, 'keep-alive request'
214
215            self.load('mirror', processes=i + 1)
216
217        for i in range(conns):
218            resp = self.post(sock=socks[i], body=body)
219
220            assert resp['body'] == body, 'keep-alive close'
221
222            self.load('mirror', processes=i + 1)
223
224    def test_python_keepalive_reconfigure_2(self):
225        self.load('mirror')
226
227        assert self.get()['status'] == 200, 'init'
228
229        body = '0123456789'
230
231        (resp, sock) = self.post(
232            headers={
233                'Host': 'localhost',
234                'Connection': 'keep-alive',
235            },
236            start=True,
237            body=body,
238            read_timeout=1,
239        )
240
241        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
242
243        self.load('empty')
244
245        assert self.get()['status'] == 200, 'init'
246
247        (resp, sock) = self.post(start=True, sock=sock, body=body)
248
249        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
250        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
251
252        assert 'success' in self.conf(
253            {"listeners": {}, "applications": {}}
254        ), 'reconfigure 2 clear configuration'
255
256        resp = self.get(sock=sock)
257
258        assert resp == {}, 'reconfigure 2 keep-alive 3'
259
260    def test_python_atexit(self):
261        self.load('atexit')
262
263        self.get()
264
265        assert 'success' in self.conf({"listeners": {}, "applications": {}})
266
267        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
268
269    def test_python_process_switch(self):
270        self.load('delayed', processes=2)
271
272        self.get(
273            headers={
274                'Host': 'localhost',
275                'Content-Length': '0',
276                'X-Delay': '5',
277                'Connection': 'close',
278            },
279            no_recv=True,
280        )
281
282        headers_delay_1 = {
283            'Connection': 'close',
284            'Host': 'localhost',
285            'Content-Length': '0',
286            'X-Delay': '1',
287        }
288
289        self.get(headers=headers_delay_1, no_recv=True)
290
291        time.sleep(0.5)
292
293        for _ in range(10):
294            self.get(headers=headers_delay_1, no_recv=True)
295
296        self.get(headers=headers_delay_1)
297
298    @pytest.mark.skip('not yet')
299    def test_python_application_start_response_exit(self):
300        self.load('start_response_exit')
301
302        assert self.get()['status'] == 500, 'start response exit'
303
304    def test_python_application_input_iter(self):
305        self.load('input_iter')
306
307        body = '''0123456789
308next line
309
310last line'''
311
312        resp = self.post(body=body)
313        assert resp['body'] == body, 'input iter'
314        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
315
316    def test_python_application_input_readline(self):
317        self.load('input_readline')
318
319        body = '''0123456789
320next line
321
322last line'''
323
324        resp = self.post(body=body)
325        assert resp['body'] == body, 'input readline'
326        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
327
328    def test_python_application_input_readline_size(self):
329        self.load('input_readline_size')
330
331        body = '''0123456789
332next line
333
334last line'''
335
336        assert self.post(body=body)['body'] == body, 'input readline size'
337        assert (
338            self.post(body='0123')['body'] == '0123'
339        ), 'input readline size less'
340
341    def test_python_application_input_readlines(self):
342        self.load('input_readlines')
343
344        body = '''0123456789
345next line
346
347last line'''
348
349        resp = self.post(body=body)
350        assert resp['body'] == body, 'input readlines'
351        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
352
353    def test_python_application_input_readlines_huge(self):
354        self.load('input_readlines')
355
356        body = (
357            '''0123456789 abcdefghi
358next line: 0123456789 abcdefghi
359
360last line: 987654321
361'''
362            * 512
363        )
364
365        assert (
366            self.post(body=body, read_buffer_size=16384)['body'] == body
367        ), 'input readlines huge'
368
369    def test_python_application_input_read_length(self):
370        self.load('input_read_length')
371
372        body = '0123456789'
373
374        resp = self.post(
375            headers={
376                'Host': 'localhost',
377                'Input-Length': '5',
378                'Connection': 'close',
379            },
380            body=body,
381        )
382
383        assert resp['body'] == body[:5], 'input read length lt body'
384
385        resp = self.post(
386            headers={
387                'Host': 'localhost',
388                'Input-Length': '15',
389                'Connection': 'close',
390            },
391            body=body,
392        )
393
394        assert resp['body'] == body, 'input read length gt body'
395
396        resp = self.post(
397            headers={
398                'Host': 'localhost',
399                'Input-Length': '0',
400                'Connection': 'close',
401            },
402            body=body,
403        )
404
405        assert resp['body'] == '', 'input read length zero'
406
407        resp = self.post(
408            headers={
409                'Host': 'localhost',
410                'Input-Length': '-1',
411                'Connection': 'close',
412            },
413            body=body,
414        )
415
416        assert resp['body'] == body, 'input read length negative'
417
418    @pytest.mark.skip('not yet')
419    def test_python_application_errors_write(self):
420        self.load('errors_write')
421
422        self.get()
423
424        assert (
425            self.wait_for_record(r'\[error\].+Error in application\.')
426            is not None
427        ), 'errors write'
428
429    def test_python_application_body_array(self):
430        self.load('body_array')
431
432        assert self.get()['body'] == '0123456789', 'body array'
433
434    def test_python_application_body_io(self):
435        self.load('body_io')
436
437        assert self.get()['body'] == '0123456789', 'body io'
438
439    def test_python_application_body_io_file(self):
440        self.load('body_io_file')
441
442        assert self.get()['body'] == 'body\n', 'body io file'
443
444    @pytest.mark.skip('not yet')
445    def test_python_application_syntax_error(self, skip_alert):
446        skip_alert(r'Python failed to import module "wsgi"')
447        self.load('syntax_error')
448
449        assert self.get()['status'] == 500, 'syntax error'
450
451    def test_python_application_loading_error(self, skip_alert):
452        skip_alert(r'Python failed to import module "blah"')
453
454        self.load('empty', module="blah")
455
456        assert self.get()['status'] == 503, 'loading error'
457
458    def test_python_application_close(self):
459        self.load('close')
460
461        self.get()
462
463        assert self.wait_for_record(r'Close called\.') is not None, 'close'
464
465    def test_python_application_close_error(self):
466        self.load('close_error')
467
468        self.get()
469
470        assert (
471            self.wait_for_record(r'Close called\.') is not None
472        ), 'close error'
473
474    def test_python_application_not_iterable(self):
475        self.load('not_iterable')
476
477        self.get()
478
479        assert (
480            self.wait_for_record(
481                r'\[error\].+the application returned not an iterable object'
482            )
483            is not None
484        ), 'not iterable'
485
486    def test_python_application_write(self):
487        self.load('write')
488
489        assert self.get()['body'] == '0123456789', 'write'
490
491    def test_python_application_threading(self):
492        """wait_for_record() timeouts after 5s while every thread works at
493        least 3s.  So without releasing GIL test should fail.
494        """
495
496        self.load('threading')
497
498        for _ in range(10):
499            self.get(no_recv=True)
500
501        assert (
502            self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None
503        ), 'last thread finished'
504
505    def test_python_application_iter_exception(self):
506        self.load('iter_exception')
507
508        # Default request doesn't lead to the exception.
509
510        resp = self.get(
511            headers={
512                'Host': 'localhost',
513                'X-Skip': '9',
514                'X-Chunked': '1',
515                'Connection': 'close',
516            }
517        )
518        assert resp['status'] == 200, 'status'
519        assert resp['body'] == 'XXXXXXX', 'body'
520
521        # Exception before start_response().
522
523        assert self.get()['status'] == 503, 'error'
524
525        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
526        assert (
527            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
528            is not None
529        ), 'first exception raise'
530        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
531
532        # Exception after start_response(), before first write().
533
534        assert (
535            self.get(
536                headers={
537                    'Host': 'localhost',
538                    'X-Skip': '1',
539                    'Connection': 'close',
540                }
541            )['status']
542            == 503
543        ), 'error 2'
544
545        assert (
546            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
547            is not None
548        ), 'exception raise second'
549        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
550
551        # Exception after first write(), before first __next__().
552
553        _, sock = self.get(
554            headers={
555                'Host': 'localhost',
556                'X-Skip': '2',
557                'Connection': 'keep-alive',
558            },
559            start=True,
560        )
561
562        assert (
563            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
564            is not None
565        ), 'exception raise third'
566        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
567
568        assert self.get(sock=sock) == {}, 'closed connection'
569
570        # Exception after first write(), before first __next__(),
571        # chunked (incomplete body).
572
573        resp = self.get(
574            headers={
575                'Host': 'localhost',
576                'X-Skip': '2',
577                'X-Chunked': '1',
578                'Connection': 'close',
579            },
580            raw_resp=True,
581        )
582        if resp:
583            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
584        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
585
586        # Exception in __next__().
587
588        _, sock = self.get(
589            headers={
590                'Host': 'localhost',
591                'X-Skip': '3',
592                'Connection': 'keep-alive',
593            },
594            start=True,
595        )
596
597        assert (
598            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
599            is not None
600        ), 'exception raise next'
601        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
602
603        assert self.get(sock=sock) == {}, 'closed connection 2'
604
605        # Exception in __next__(), chunked (incomplete body).
606
607        resp = self.get(
608            headers={
609                'Host': 'localhost',
610                'X-Skip': '3',
611                'X-Chunked': '1',
612                'Connection': 'close',
613            },
614            raw_resp=True,
615        )
616        if resp:
617            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
618        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
619
620        # Exception before start_response() and in close().
621
622        assert (
623            self.get(
624                headers={
625                    'Host': 'localhost',
626                    'X-Not-Skip-Close': '1',
627                    'Connection': 'close',
628                }
629            )['status']
630            == 503
631        ), 'error'
632
633        assert (
634            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
635            is not None
636        ), 'exception raise close'
637        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
638
639    def test_python_user_group(self, is_su):
640        if not is_su:
641            pytest.skip('requires root')
642
643        nobody_uid = pwd.getpwnam('nobody').pw_uid
644
645        group = 'nobody'
646
647        try:
648            group_id = grp.getgrnam(group).gr_gid
649        except KeyError:
650            group = 'nogroup'
651            group_id = grp.getgrnam(group).gr_gid
652
653        self.load('user_group')
654
655        obj = self.getjson()['body']
656        assert obj['UID'] == nobody_uid, 'nobody uid'
657        assert obj['GID'] == group_id, 'nobody gid'
658
659        self.load('user_group', user='nobody')
660
661        obj = self.getjson()['body']
662        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
663        assert obj['GID'] == group_id, 'nobody gid user=nobody'
664
665        self.load('user_group', user='nobody', group=group)
666
667        obj = self.getjson()['body']
668        assert obj['UID'] == nobody_uid, (
669            'nobody uid user=nobody group=%s' % group
670        )
671
672        assert obj['GID'] == group_id, 'nobody gid user=nobody group=%s' % group
673
674        self.load('user_group', group=group)
675
676        obj = self.getjson()['body']
677        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
678
679        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
680
681        self.load('user_group', user='root')
682
683        obj = self.getjson()['body']
684        assert obj['UID'] == 0, 'root uid user=root'
685        assert obj['GID'] == 0, 'root gid user=root'
686
687        group = 'root'
688
689        try:
690            grp.getgrnam(group)
691            group = True
692        except KeyError:
693            group = False
694
695        if group:
696            self.load('user_group', user='root', group='root')
697
698            obj = self.getjson()['body']
699            assert obj['UID'] == 0, 'root uid user=root group=root'
700            assert obj['GID'] == 0, 'root gid user=root group=root'
701
702            self.load('user_group', group='root')
703
704            obj = self.getjson()['body']
705            assert obj['UID'] == nobody_uid, 'root uid group=root'
706            assert obj['GID'] == 0, 'root gid group=root'
707
708    def test_python_application_callable(self, skip_alert):
709        skip_alert(r'Python failed to get "blah" from module')
710        self.load('callable')
711
712        assert self.get()['status'] == 204, 'default application response'
713
714        self.load('callable', callable="app")
715
716        assert self.get()['status'] == 200, 'callable response'
717
718        self.load('callable', callable="blah")
719
720        assert self.get()['status'] not in [200, 204], 'callable response inv'
721
722    def test_python_application_path(self):
723        self.load('path')
724
725        def set_path(path):
726            assert 'success' in self.conf(path, 'applications/path/path')
727
728        def get_path():
729            return self.get()['body'].split(os.pathsep)
730
731        default_path = self.conf_get('/config/applications/path/path')
732        assert 'success' in self.conf(
733            {"PYTHONPATH": default_path},
734            '/config/applications/path/environment',
735        )
736
737        self.conf_delete('/config/applications/path/path')
738        sys_path = get_path()
739
740        set_path('"/blah"')
741        assert ['/blah', *sys_path] == get_path(), 'check path'
742
743        set_path('"/new"')
744        assert ['/new', *sys_path] == get_path(), 'check path update'
745
746        set_path('["/blah1", "/blah2"]')
747        assert [
748            '/blah1',
749            '/blah2',
750            *sys_path,
751        ] == get_path(), 'check path array'
752
753    def test_python_application_path_invalid(self):
754        self.load('path')
755
756        def check_path(path):
757            assert 'error' in self.conf(path, 'applications/path/path')
758
759        check_path('{}')
760        check_path('["/blah", []]')
761
762    def test_python_application_threads(self):
763        self.load('threads', threads=4)
764
765        socks = []
766
767        for i in range(4):
768            sock = self.get(
769                headers={
770                    'Host': 'localhost',
771                    'X-Delay': '2',
772                    'Connection': 'close',
773                },
774                no_recv=True,
775            )
776
777            socks.append(sock)
778
779        threads = set()
780
781        for sock in socks:
782            resp = self.recvall(sock).decode('utf-8')
783
784            self.log_in(resp)
785
786            resp = self._resp_to_dict(resp)
787
788            assert resp['status'] == 200, 'status'
789
790            threads.add(resp['headers']['X-Thread'])
791
792            assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread'
793
794            sock.close()
795
796        assert len(socks) == len(threads), 'threads differs'
797