xref: /unit/test/test_python_application.py (revision 1850:839024ce4a6a)
1import grp
2import os
3import pwd
4import re
5import time
6
7import pytest
8
9from unit.applications.lang.python import TestApplicationPython
10
11
12class TestPythonApplication(TestApplicationPython):
13    prerequisites = {'modules': {'python': 'all'}}
14
15    def test_python_application_variables(self):
16        self.load('variables')
17
18        body = 'Test body string.'
19
20        resp = self.http(
21            b"""POST / HTTP/1.1
22Host: localhost
23Content-Length: %d
24Custom-Header: blah
25Custom-hEader: Blah
26Content-Type: text/html
27Connection: close
28custom-header: BLAH
29
30%s"""
31            % (len(body), body.encode()),
32            raw=True,
33        )
34
35        assert resp['status'] == 200, 'status'
36        headers = resp['headers']
37        header_server = headers.pop('Server')
38        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
39        assert (
40            headers.pop('Server-Software') == header_server
41        ), 'server software header'
42
43        date = headers.pop('Date')
44        assert date[-4:] == ' GMT', 'date header timezone'
45        assert (
46            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
47        ), 'date header'
48
49        assert headers == {
50            'Connection': 'close',
51            'Content-Length': str(len(body)),
52            'Content-Type': 'text/html',
53            'Request-Method': 'POST',
54            'Request-Uri': '/',
55            'Http-Host': 'localhost',
56            'Server-Protocol': 'HTTP/1.1',
57            'Custom-Header': 'blah, Blah, BLAH',
58            'Wsgi-Version': '(1, 0)',
59            'Wsgi-Url-Scheme': 'http',
60            'Wsgi-Multithread': 'False',
61            'Wsgi-Multiprocess': 'True',
62            'Wsgi-Run-Once': 'False',
63        }, 'headers'
64        assert resp['body'] == body, 'body'
65
66    def test_python_application_query_string(self):
67        self.load('query_string')
68
69        resp = self.get(url='/?var1=val1&var2=val2')
70
71        assert (
72            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
73        ), 'Query-String header'
74
75    def test_python_application_query_string_space(self):
76        self.load('query_string')
77
78        resp = self.get(url='/ ?var1=val1&var2=val2')
79        assert (
80            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
81        ), 'Query-String space'
82
83        resp = self.get(url='/ %20?var1=val1&var2=val2')
84        assert (
85            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
86        ), 'Query-String space 2'
87
88        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
89        assert (
90            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
91        ), 'Query-String space 3'
92
93        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
94        assert (
95            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
96        ), 'Query-String space 4'
97
98    def test_python_application_query_string_empty(self):
99        self.load('query_string')
100
101        resp = self.get(url='/?')
102
103        assert resp['status'] == 200, 'query string empty status'
104        assert resp['headers']['Query-String'] == '', 'query string empty'
105
106    def test_python_application_query_string_absent(self):
107        self.load('query_string')
108
109        resp = self.get()
110
111        assert resp['status'] == 200, 'query string absent status'
112        assert resp['headers']['Query-String'] == '', 'query string absent'
113
114    @pytest.mark.skip('not yet')
115    def test_python_application_server_port(self):
116        self.load('server_port')
117
118        assert (
119            self.get()['headers']['Server-Port'] == '7080'
120        ), 'Server-Port header'
121
122    @pytest.mark.skip('not yet')
123    def test_python_application_working_directory_invalid(self):
124        self.load('empty')
125
126        assert 'success' in self.conf(
127            '"/blah"', 'applications/empty/working_directory'
128        ), 'configure invalid working_directory'
129
130        assert self.get()['status'] == 500, 'status'
131
132    def test_python_application_204_transfer_encoding(self):
133        self.load('204_no_content')
134
135        assert (
136            'Transfer-Encoding' not in self.get()['headers']
137        ), '204 header transfer encoding'
138
139    def test_python_application_ctx_iter_atexit(self):
140        self.load('ctx_iter_atexit')
141
142        resp = self.post(
143            headers={
144                'Host': 'localhost',
145                'Connection': 'close',
146                'Content-Type': 'text/html',
147            },
148            body='0123456789',
149        )
150
151        assert resp['status'] == 200, 'ctx iter status'
152        assert resp['body'] == '0123456789', 'ctx iter body'
153
154        assert 'success' in self.conf({"listeners": {}, "applications": {}})
155
156        assert (
157            self.wait_for_record(r'RuntimeError') is not None
158        ), 'ctx iter atexit'
159
160    def test_python_keepalive_body(self):
161        self.load('mirror')
162
163        assert self.get()['status'] == 200, 'init'
164
165        body = '0123456789' * 500
166        (resp, sock) = self.post(
167            headers={
168                'Host': 'localhost',
169                'Connection': 'keep-alive',
170                'Content-Type': 'text/html',
171            },
172            start=True,
173            body=body,
174            read_timeout=1,
175        )
176
177        assert resp['body'] == body, 'keep-alive 1'
178
179        body = '0123456789'
180        resp = self.post(
181            headers={
182                'Host': 'localhost',
183                'Connection': 'close',
184                'Content-Type': 'text/html',
185            },
186            sock=sock,
187            body=body,
188        )
189
190        assert resp['body'] == body, 'keep-alive 2'
191
192    def test_python_keepalive_reconfigure(self):
193        self.load('mirror')
194
195        assert self.get()['status'] == 200, 'init'
196
197        body = '0123456789'
198        conns = 3
199        socks = []
200
201        for i in range(conns):
202            (resp, sock) = self.post(
203                headers={
204                    'Host': 'localhost',
205                    'Connection': 'keep-alive',
206                    'Content-Type': 'text/html',
207                },
208                start=True,
209                body=body,
210                read_timeout=1,
211            )
212
213            assert resp['body'] == body, 'keep-alive open'
214
215            self.load('mirror', processes=i + 1)
216
217            socks.append(sock)
218
219        for i in range(conns):
220            (resp, sock) = self.post(
221                headers={
222                    'Host': 'localhost',
223                    'Connection': 'keep-alive',
224                    'Content-Type': 'text/html',
225                },
226                start=True,
227                sock=socks[i],
228                body=body,
229                read_timeout=1,
230            )
231
232            assert resp['body'] == body, 'keep-alive request'
233
234            self.load('mirror', processes=i + 1)
235
236        for i in range(conns):
237            resp = self.post(
238                headers={
239                    'Host': 'localhost',
240                    'Connection': 'close',
241                    'Content-Type': 'text/html',
242                },
243                sock=socks[i],
244                body=body,
245            )
246
247            assert resp['body'] == body, 'keep-alive close'
248
249            self.load('mirror', processes=i + 1)
250
251    def test_python_keepalive_reconfigure_2(self):
252        self.load('mirror')
253
254        assert self.get()['status'] == 200, 'init'
255
256        body = '0123456789'
257
258        (resp, sock) = self.post(
259            headers={
260                'Host': 'localhost',
261                'Connection': 'keep-alive',
262                'Content-Type': 'text/html',
263            },
264            start=True,
265            body=body,
266            read_timeout=1,
267        )
268
269        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
270
271        self.load('empty')
272
273        assert self.get()['status'] == 200, 'init'
274
275        (resp, sock) = self.post(
276            headers={
277                'Host': 'localhost',
278                'Connection': 'close',
279                'Content-Type': 'text/html',
280            },
281            start=True,
282            sock=sock,
283            body=body,
284        )
285
286        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
287        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
288
289        assert 'success' in self.conf(
290            {"listeners": {}, "applications": {}}
291        ), 'reconfigure 2 clear configuration'
292
293        resp = self.get(sock=sock)
294
295        assert resp == {}, 'reconfigure 2 keep-alive 3'
296
297    def test_python_keepalive_reconfigure_3(self):
298        self.load('empty')
299
300        assert self.get()['status'] == 200, 'init'
301
302        (_, sock) = self.http(
303            b"""GET / HTTP/1.1
304""",
305            start=True,
306            raw=True,
307            no_recv=True,
308        )
309
310        assert self.get()['status'] == 200
311
312        assert 'success' in self.conf(
313            {"listeners": {}, "applications": {}}
314        ), 'reconfigure 3 clear configuration'
315
316        resp = self.http(
317            b"""Host: localhost
318Connection: close
319
320""",
321            sock=sock,
322            raw=True,
323        )
324
325        assert resp['status'] == 200, 'reconfigure 3'
326
327    def test_python_atexit(self):
328        self.load('atexit')
329
330        self.get()
331
332        assert 'success' in self.conf({"listeners": {}, "applications": {}})
333
334        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
335
336    def test_python_process_switch(self):
337        self.load('delayed', processes=2)
338
339        self.get(
340            headers={
341                'Host': 'localhost',
342                'Content-Length': '0',
343                'X-Delay': '5',
344                'Connection': 'close',
345            },
346            no_recv=True,
347        )
348
349        headers_delay_1 = {
350            'Connection': 'close',
351            'Host': 'localhost',
352            'Content-Length': '0',
353            'X-Delay': '1',
354        }
355
356        self.get(headers=headers_delay_1, no_recv=True)
357
358        time.sleep(0.5)
359
360        for _ in range(10):
361            self.get(headers=headers_delay_1, no_recv=True)
362
363        self.get(headers=headers_delay_1)
364
365    @pytest.mark.skip('not yet')
366    def test_python_application_start_response_exit(self):
367        self.load('start_response_exit')
368
369        assert self.get()['status'] == 500, 'start response exit'
370
371    def test_python_application_input_iter(self):
372        self.load('input_iter')
373
374        body = '''0123456789
375next line
376
377last line'''
378
379        resp = self.post(body=body)
380        assert resp['body'] == body, 'input iter'
381        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
382
383    def test_python_application_input_readline(self):
384        self.load('input_readline')
385
386        body = '''0123456789
387next line
388
389last line'''
390
391        resp = self.post(body=body)
392        assert resp['body'] == body, 'input readline'
393        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
394
395    def test_python_application_input_readline_size(self):
396        self.load('input_readline_size')
397
398        body = '''0123456789
399next line
400
401last line'''
402
403        assert self.post(body=body)['body'] == body, 'input readline size'
404        assert (
405            self.post(body='0123')['body'] == '0123'
406        ), 'input readline size less'
407
408    def test_python_application_input_readlines(self):
409        self.load('input_readlines')
410
411        body = '''0123456789
412next line
413
414last line'''
415
416        resp = self.post(body=body)
417        assert resp['body'] == body, 'input readlines'
418        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
419
420    def test_python_application_input_readlines_huge(self):
421        self.load('input_readlines')
422
423        body = (
424            '''0123456789 abcdefghi
425next line: 0123456789 abcdefghi
426
427last line: 987654321
428'''
429            * 512
430        )
431
432        assert (
433            self.post(body=body, read_buffer_size=16384)['body'] == body
434        ), 'input readlines huge'
435
436    def test_python_application_input_read_length(self):
437        self.load('input_read_length')
438
439        body = '0123456789'
440
441        resp = self.post(
442            headers={
443                'Host': 'localhost',
444                'Input-Length': '5',
445                'Connection': 'close',
446            },
447            body=body,
448        )
449
450        assert resp['body'] == body[:5], 'input read length lt body'
451
452        resp = self.post(
453            headers={
454                'Host': 'localhost',
455                'Input-Length': '15',
456                'Connection': 'close',
457            },
458            body=body,
459        )
460
461        assert resp['body'] == body, 'input read length gt body'
462
463        resp = self.post(
464            headers={
465                'Host': 'localhost',
466                'Input-Length': '0',
467                'Connection': 'close',
468            },
469            body=body,
470        )
471
472        assert resp['body'] == '', 'input read length zero'
473
474        resp = self.post(
475            headers={
476                'Host': 'localhost',
477                'Input-Length': '-1',
478                'Connection': 'close',
479            },
480            body=body,
481        )
482
483        assert resp['body'] == body, 'input read length negative'
484
485    @pytest.mark.skip('not yet')
486    def test_python_application_errors_write(self):
487        self.load('errors_write')
488
489        self.get()
490
491        assert (
492            self.wait_for_record(r'\[error\].+Error in application\.')
493            is not None
494        ), 'errors write'
495
496    def test_python_application_body_array(self):
497        self.load('body_array')
498
499        assert self.get()['body'] == '0123456789', 'body array'
500
501    def test_python_application_body_io(self):
502        self.load('body_io')
503
504        assert self.get()['body'] == '0123456789', 'body io'
505
506    def test_python_application_body_io_file(self):
507        self.load('body_io_file')
508
509        assert self.get()['body'] == 'body\n', 'body io file'
510
511    @pytest.mark.skip('not yet')
512    def test_python_application_syntax_error(self, skip_alert):
513        skip_alert(r'Python failed to import module "wsgi"')
514        self.load('syntax_error')
515
516        assert self.get()['status'] == 500, 'syntax error'
517
518    def test_python_application_loading_error(self, skip_alert):
519        skip_alert(r'Python failed to import module "blah"')
520
521        self.load('empty', module="blah")
522
523        assert self.get()['status'] == 503, 'loading error'
524
525    def test_python_application_close(self):
526        self.load('close')
527
528        self.get()
529
530        assert self.wait_for_record(r'Close called\.') is not None, 'close'
531
532    def test_python_application_close_error(self):
533        self.load('close_error')
534
535        self.get()
536
537        assert (
538            self.wait_for_record(r'Close called\.') is not None
539        ), 'close error'
540
541    def test_python_application_not_iterable(self):
542        self.load('not_iterable')
543
544        self.get()
545
546        assert (
547            self.wait_for_record(
548                r'\[error\].+the application returned not an iterable object'
549            )
550            is not None
551        ), 'not iterable'
552
553    def test_python_application_write(self):
554        self.load('write')
555
556        assert self.get()['body'] == '0123456789', 'write'
557
558    def test_python_application_threading(self):
559        """wait_for_record() timeouts after 5s while every thread works at
560        least 3s.  So without releasing GIL test should fail.
561        """
562
563        self.load('threading')
564
565        for _ in range(10):
566            self.get(no_recv=True)
567
568        assert (
569            self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None
570        ), 'last thread finished'
571
572    def test_python_application_iter_exception(self):
573        self.load('iter_exception')
574
575        # Default request doesn't lead to the exception.
576
577        resp = self.get(
578            headers={
579                'Host': 'localhost',
580                'X-Skip': '9',
581                'X-Chunked': '1',
582                'Connection': 'close',
583            }
584        )
585        assert resp['status'] == 200, 'status'
586        assert resp['body'] == 'XXXXXXX', 'body'
587
588        # Exception before start_response().
589
590        assert self.get()['status'] == 503, 'error'
591
592        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
593        assert (
594            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
595            is not None
596        ), 'first exception raise'
597        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
598
599        # Exception after start_response(), before first write().
600
601        assert (
602            self.get(
603                headers={
604                    'Host': 'localhost',
605                    'X-Skip': '1',
606                    'Connection': 'close',
607                }
608            )['status']
609            == 503
610        ), 'error 2'
611
612        assert (
613            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
614            is not None
615        ), 'exception raise second'
616        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
617
618        # Exception after first write(), before first __next__().
619
620        _, sock = self.get(
621            headers={
622                'Host': 'localhost',
623                'X-Skip': '2',
624                'Connection': 'keep-alive',
625            },
626            start=True,
627        )
628
629        assert (
630            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
631            is not None
632        ), 'exception raise third'
633        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
634
635        assert self.get(sock=sock) == {}, 'closed connection'
636
637        # Exception after first write(), before first __next__(),
638        # chunked (incomplete body).
639
640        resp = self.get(
641            headers={
642                'Host': 'localhost',
643                'X-Skip': '2',
644                'X-Chunked': '1',
645                'Connection': 'close',
646            },
647            raw_resp=True,
648        )
649        if resp:
650            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
651        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
652
653        # Exception in __next__().
654
655        _, sock = self.get(
656            headers={
657                'Host': 'localhost',
658                'X-Skip': '3',
659                'Connection': 'keep-alive',
660            },
661            start=True,
662        )
663
664        assert (
665            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
666            is not None
667        ), 'exception raise next'
668        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
669
670        assert self.get(sock=sock) == {}, 'closed connection 2'
671
672        # Exception in __next__(), chunked (incomplete body).
673
674        resp = self.get(
675            headers={
676                'Host': 'localhost',
677                'X-Skip': '3',
678                'X-Chunked': '1',
679                'Connection': 'close',
680            },
681            raw_resp=True,
682        )
683        if resp:
684            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
685        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
686
687        # Exception before start_response() and in close().
688
689        assert (
690            self.get(
691                headers={
692                    'Host': 'localhost',
693                    'X-Not-Skip-Close': '1',
694                    'Connection': 'close',
695                }
696            )['status']
697            == 503
698        ), 'error'
699
700        assert (
701            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
702            is not None
703        ), 'exception raise close'
704        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
705
706    def test_python_user_group(self, is_su):
707        if not is_su:
708            pytest.skip('requires root')
709
710        nobody_uid = pwd.getpwnam('nobody').pw_uid
711
712        group = 'nobody'
713
714        try:
715            group_id = grp.getgrnam(group).gr_gid
716        except KeyError:
717            group = 'nogroup'
718            group_id = grp.getgrnam(group).gr_gid
719
720        self.load('user_group')
721
722        obj = self.getjson()['body']
723        assert obj['UID'] == nobody_uid, 'nobody uid'
724        assert obj['GID'] == group_id, 'nobody gid'
725
726        self.load('user_group', user='nobody')
727
728        obj = self.getjson()['body']
729        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
730        assert obj['GID'] == group_id, 'nobody gid user=nobody'
731
732        self.load('user_group', user='nobody', group=group)
733
734        obj = self.getjson()['body']
735        assert obj['UID'] == nobody_uid, (
736            'nobody uid user=nobody group=%s' % group
737        )
738
739        assert obj['GID'] == group_id, (
740            'nobody gid user=nobody group=%s' % group
741        )
742
743        self.load('user_group', group=group)
744
745        obj = self.getjson()['body']
746        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
747
748        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
749
750        self.load('user_group', user='root')
751
752        obj = self.getjson()['body']
753        assert obj['UID'] == 0, 'root uid user=root'
754        assert obj['GID'] == 0, 'root gid user=root'
755
756        group = 'root'
757
758        try:
759            grp.getgrnam(group)
760            group = True
761        except KeyError:
762            group = False
763
764        if group:
765            self.load('user_group', user='root', group='root')
766
767            obj = self.getjson()['body']
768            assert obj['UID'] == 0, 'root uid user=root group=root'
769            assert obj['GID'] == 0, 'root gid user=root group=root'
770
771            self.load('user_group', group='root')
772
773            obj = self.getjson()['body']
774            assert obj['UID'] == nobody_uid, 'root uid group=root'
775            assert obj['GID'] == 0, 'root gid group=root'
776
777    def test_python_application_callable(self, skip_alert):
778        skip_alert(r'Python failed to get "blah" from module')
779        self.load('callable')
780
781        assert self.get()['status'] == 204, 'default application response'
782
783        self.load('callable', callable="app")
784
785        assert self.get()['status'] == 200, 'callable response'
786
787        self.load('callable', callable="blah")
788
789        assert self.get()['status'] not in [200, 204], 'callable response inv'
790
791    def test_python_application_path(self):
792        self.load('path')
793
794        def set_path(path):
795            assert 'success' in self.conf(path, 'applications/path/path')
796
797        def get_path():
798            return self.get()['body'].split(os.pathsep)
799
800        default_path = self.conf_get('/config/applications/path/path')
801        assert 'success' in self.conf(
802            {"PYTHONPATH": default_path},
803            '/config/applications/path/environment',
804        )
805
806        self.conf_delete('/config/applications/path/path')
807        sys_path = get_path()
808
809        set_path('"/blah"')
810        assert ['/blah', *sys_path] == get_path(), 'check path'
811
812        set_path('"/new"')
813        assert ['/new', *sys_path] == get_path(), 'check path update'
814
815        set_path('["/blah1", "/blah2"]')
816        assert [
817            '/blah1',
818            '/blah2',
819            *sys_path,
820        ] == get_path(), 'check path array'
821
822    def test_python_application_path_invalid(self):
823        self.load('path')
824
825        def check_path(path):
826            assert 'error' in self.conf(path, 'applications/path/path')
827
828        check_path('{}')
829        check_path('["/blah", []]')
830
831    def test_python_application_threads(self):
832        self.load('threads', threads=4)
833
834        socks = []
835
836        for i in range(4):
837            (_, sock) = self.get(
838                headers={
839                    'Host': 'localhost',
840                    'X-Delay': '2',
841                    'Connection': 'close',
842                },
843                no_recv=True,
844                start=True,
845            )
846
847            socks.append(sock)
848
849        threads = set()
850
851        for sock in socks:
852            resp = self.recvall(sock).decode('utf-8')
853
854            self.log_in(resp)
855
856            resp = self._resp_to_dict(resp)
857
858            assert resp['status'] == 200, 'status'
859
860            threads.add(resp['headers']['X-Thread'])
861
862            assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread'
863
864            sock.close()
865
866        assert len(socks) == len(threads), 'threads differs'
867