xref: /unit/test/test_python_application.py (revision 1804:ed698edaaf1e)
1import grp
2import os
3import pwd
4import re
5import time
6
7import pytest
8from unit.applications.lang.python import TestApplicationPython
9from unit.option import option
10
11
12class TestPythonApplication(TestApplicationPython):
13    prerequisites = {'modules': {'python': 'all'}}
14
15    def findall(self, pattern):
16        with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f:
17            return re.findall(pattern, f.read())
18
19    def test_python_application_variables(self):
20        self.load('variables')
21
22        body = 'Test body string.'
23
24        resp = self.http(
25            b"""POST / HTTP/1.1
26Host: localhost
27Content-Length: %d
28Custom-Header: blah
29Custom-hEader: Blah
30Content-Type: text/html
31Connection: close
32custom-header: BLAH
33
34%s""" % (len(body), body.encode()),
35            raw=True,
36        )
37
38        assert resp['status'] == 200, 'status'
39        headers = resp['headers']
40        header_server = headers.pop('Server')
41        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
42        assert (
43            headers.pop('Server-Software') == header_server
44        ), 'server software header'
45
46        date = headers.pop('Date')
47        assert date[-4:] == ' GMT', 'date header timezone'
48        assert (
49            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
50        ), 'date header'
51
52        assert headers == {
53            'Connection': 'close',
54            'Content-Length': str(len(body)),
55            'Content-Type': 'text/html',
56            'Request-Method': 'POST',
57            'Request-Uri': '/',
58            'Http-Host': 'localhost',
59            'Server-Protocol': 'HTTP/1.1',
60            'Custom-Header': 'blah, Blah, BLAH',
61            'Wsgi-Version': '(1, 0)',
62            'Wsgi-Url-Scheme': 'http',
63            'Wsgi-Multithread': 'False',
64            'Wsgi-Multiprocess': 'True',
65            'Wsgi-Run-Once': 'False',
66        }, 'headers'
67        assert resp['body'] == body, 'body'
68
69    def test_python_application_query_string(self):
70        self.load('query_string')
71
72        resp = self.get(url='/?var1=val1&var2=val2')
73
74        assert (
75            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
76        ), 'Query-String header'
77
78    def test_python_application_query_string_space(self):
79        self.load('query_string')
80
81        resp = self.get(url='/ ?var1=val1&var2=val2')
82        assert (
83            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
84        ), 'Query-String space'
85
86        resp = self.get(url='/ %20?var1=val1&var2=val2')
87        assert (
88            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
89        ), 'Query-String space 2'
90
91        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
92        assert (
93            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
94        ), 'Query-String space 3'
95
96        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
97        assert (
98            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
99        ), 'Query-String space 4'
100
101    def test_python_application_query_string_empty(self):
102        self.load('query_string')
103
104        resp = self.get(url='/?')
105
106        assert resp['status'] == 200, 'query string empty status'
107        assert resp['headers']['Query-String'] == '', 'query string empty'
108
109    def test_python_application_query_string_absent(self):
110        self.load('query_string')
111
112        resp = self.get()
113
114        assert resp['status'] == 200, 'query string absent status'
115        assert resp['headers']['Query-String'] == '', 'query string absent'
116
117    @pytest.mark.skip('not yet')
118    def test_python_application_server_port(self):
119        self.load('server_port')
120
121        assert (
122            self.get()['headers']['Server-Port'] == '7080'
123        ), 'Server-Port header'
124
125    @pytest.mark.skip('not yet')
126    def test_python_application_working_directory_invalid(self):
127        self.load('empty')
128
129        assert 'success' in self.conf(
130            '"/blah"', 'applications/empty/working_directory'
131        ), 'configure invalid working_directory'
132
133        assert self.get()['status'] == 500, 'status'
134
135    def test_python_application_204_transfer_encoding(self):
136        self.load('204_no_content')
137
138        assert (
139            'Transfer-Encoding' not in self.get()['headers']
140        ), '204 header transfer encoding'
141
142    def test_python_application_ctx_iter_atexit(self):
143        self.load('ctx_iter_atexit')
144
145        resp = self.post(
146            headers={
147                'Host': 'localhost',
148                'Connection': 'close',
149                'Content-Type': 'text/html',
150            },
151            body='0123456789',
152        )
153
154        assert resp['status'] == 200, 'ctx iter status'
155        assert resp['body'] == '0123456789', 'ctx iter body'
156
157        assert 'success' in self.conf({"listeners": {}, "applications": {}})
158
159        assert (
160            self.wait_for_record(r'RuntimeError') is not None
161        ), 'ctx iter atexit'
162
163    def test_python_keepalive_body(self):
164        self.load('mirror')
165
166        assert self.get()['status'] == 200, 'init'
167
168        body = '0123456789' * 500
169        (resp, sock) = self.post(
170            headers={
171                'Host': 'localhost',
172                'Connection': 'keep-alive',
173                'Content-Type': 'text/html',
174            },
175            start=True,
176            body=body,
177            read_timeout=1,
178        )
179
180        assert resp['body'] == body, 'keep-alive 1'
181
182        body = '0123456789'
183        resp = self.post(
184            headers={
185                'Host': 'localhost',
186                'Connection': 'close',
187                'Content-Type': 'text/html',
188            },
189            sock=sock,
190            body=body,
191        )
192
193        assert resp['body'] == body, 'keep-alive 2'
194
195    def test_python_keepalive_reconfigure(self):
196        self.load('mirror')
197
198        assert self.get()['status'] == 200, 'init'
199
200        body = '0123456789'
201        conns = 3
202        socks = []
203
204        for i in range(conns):
205            (resp, sock) = self.post(
206                headers={
207                    'Host': 'localhost',
208                    'Connection': 'keep-alive',
209                    'Content-Type': 'text/html',
210                },
211                start=True,
212                body=body,
213                read_timeout=1,
214            )
215
216            assert resp['body'] == body, 'keep-alive open'
217
218            self.load('mirror', processes=i + 1)
219
220            socks.append(sock)
221
222        for i in range(conns):
223            (resp, sock) = self.post(
224                headers={
225                    'Host': 'localhost',
226                    'Connection': 'keep-alive',
227                    'Content-Type': 'text/html',
228                },
229                start=True,
230                sock=socks[i],
231                body=body,
232                read_timeout=1,
233            )
234
235            assert resp['body'] == body, 'keep-alive request'
236
237            self.load('mirror', processes=i + 1)
238
239        for i in range(conns):
240            resp = self.post(
241                headers={
242                    'Host': 'localhost',
243                    'Connection': 'close',
244                    'Content-Type': 'text/html',
245                },
246                sock=socks[i],
247                body=body,
248            )
249
250            assert resp['body'] == body, 'keep-alive close'
251
252            self.load('mirror', processes=i + 1)
253
254    def test_python_keepalive_reconfigure_2(self):
255        self.load('mirror')
256
257        assert self.get()['status'] == 200, 'init'
258
259        body = '0123456789'
260
261        (resp, sock) = self.post(
262            headers={
263                'Host': 'localhost',
264                'Connection': 'keep-alive',
265                'Content-Type': 'text/html',
266            },
267            start=True,
268            body=body,
269            read_timeout=1,
270        )
271
272        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
273
274        self.load('empty')
275
276        assert self.get()['status'] == 200, 'init'
277
278        (resp, sock) = self.post(
279            headers={
280                'Host': 'localhost',
281                'Connection': 'close',
282                'Content-Type': 'text/html',
283            },
284            start=True,
285            sock=sock,
286            body=body,
287        )
288
289        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
290        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
291
292        assert 'success' in self.conf(
293            {"listeners": {}, "applications": {}}
294        ), 'reconfigure 2 clear configuration'
295
296        resp = self.get(sock=sock)
297
298        assert resp == {}, 'reconfigure 2 keep-alive 3'
299
300    def test_python_keepalive_reconfigure_3(self):
301        self.load('empty')
302
303        assert self.get()['status'] == 200, 'init'
304
305        (_, sock) = self.http(
306            b"""GET / HTTP/1.1
307""",
308            start=True,
309            raw=True,
310            no_recv=True,
311        )
312
313        assert self.get()['status'] == 200
314
315        assert 'success' in self.conf(
316            {"listeners": {}, "applications": {}}
317        ), 'reconfigure 3 clear configuration'
318
319        resp = self.http(
320            b"""Host: localhost
321Connection: close
322
323""",
324            sock=sock,
325            raw=True,
326        )
327
328        assert resp['status'] == 200, 'reconfigure 3'
329
330    def test_python_atexit(self):
331        self.load('atexit')
332
333        self.get()
334
335        assert 'success' in self.conf({"listeners": {}, "applications": {}})
336
337        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
338
339    def test_python_process_switch(self):
340        self.load('delayed', processes=2)
341
342        self.get(
343            headers={
344                'Host': 'localhost',
345                'Content-Length': '0',
346                'X-Delay': '5',
347                'Connection': 'close',
348            },
349            no_recv=True,
350        )
351
352        headers_delay_1 = {
353            'Connection': 'close',
354            'Host': 'localhost',
355            'Content-Length': '0',
356            'X-Delay': '1',
357        }
358
359        self.get(headers=headers_delay_1, no_recv=True)
360
361        time.sleep(0.5)
362
363        for _ in range(10):
364            self.get(headers=headers_delay_1, no_recv=True)
365
366        self.get(headers=headers_delay_1)
367
368    @pytest.mark.skip('not yet')
369    def test_python_application_start_response_exit(self):
370        self.load('start_response_exit')
371
372        assert self.get()['status'] == 500, 'start response exit'
373
374    def test_python_application_input_iter(self):
375        self.load('input_iter')
376
377        body = '''0123456789
378next line
379
380last line'''
381
382        resp = self.post(body=body)
383        assert resp['body'] == body, 'input iter'
384        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
385
386    def test_python_application_input_readline(self):
387        self.load('input_readline')
388
389        body = '''0123456789
390next line
391
392last line'''
393
394        resp = self.post(body=body)
395        assert resp['body'] == body, 'input readline'
396        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
397
398    def test_python_application_input_readline_size(self):
399        self.load('input_readline_size')
400
401        body = '''0123456789
402next line
403
404last line'''
405
406        assert self.post(body=body)['body'] == body, 'input readline size'
407        assert (
408            self.post(body='0123')['body'] == '0123'
409        ), 'input readline size less'
410
411    def test_python_application_input_readlines(self):
412        self.load('input_readlines')
413
414        body = '''0123456789
415next line
416
417last line'''
418
419        resp = self.post(body=body)
420        assert resp['body'] == body, 'input readlines'
421        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
422
423    def test_python_application_input_readlines_huge(self):
424        self.load('input_readlines')
425
426        body = (
427            '''0123456789 abcdefghi
428next line: 0123456789 abcdefghi
429
430last line: 987654321
431'''
432            * 512
433        )
434
435        assert (
436            self.post(body=body, read_buffer_size=16384)['body'] == body
437        ), 'input readlines huge'
438
439    def test_python_application_input_read_length(self):
440        self.load('input_read_length')
441
442        body = '0123456789'
443
444        resp = self.post(
445            headers={
446                'Host': 'localhost',
447                'Input-Length': '5',
448                'Connection': 'close',
449            },
450            body=body,
451        )
452
453        assert resp['body'] == body[:5], 'input read length lt body'
454
455        resp = self.post(
456            headers={
457                'Host': 'localhost',
458                'Input-Length': '15',
459                'Connection': 'close',
460            },
461            body=body,
462        )
463
464        assert resp['body'] == body, 'input read length gt body'
465
466        resp = self.post(
467            headers={
468                'Host': 'localhost',
469                'Input-Length': '0',
470                'Connection': 'close',
471            },
472            body=body,
473        )
474
475        assert resp['body'] == '', 'input read length zero'
476
477        resp = self.post(
478            headers={
479                'Host': 'localhost',
480                'Input-Length': '-1',
481                'Connection': 'close',
482            },
483            body=body,
484        )
485
486        assert resp['body'] == body, 'input read length negative'
487
488    @pytest.mark.skip('not yet')
489    def test_python_application_errors_write(self):
490        self.load('errors_write')
491
492        self.get()
493
494        assert (
495            self.wait_for_record(r'\[error\].+Error in application\.')
496            is not None
497        ), 'errors write'
498
499    def test_python_application_body_array(self):
500        self.load('body_array')
501
502        assert self.get()['body'] == '0123456789', 'body array'
503
504    def test_python_application_body_io(self):
505        self.load('body_io')
506
507        assert self.get()['body'] == '0123456789', 'body io'
508
509    def test_python_application_body_io_file(self):
510        self.load('body_io_file')
511
512        assert self.get()['body'] == 'body\n', 'body io file'
513
514    @pytest.mark.skip('not yet')
515    def test_python_application_syntax_error(self, skip_alert):
516        skip_alert(r'Python failed to import module "wsgi"')
517        self.load('syntax_error')
518
519        assert self.get()['status'] == 500, 'syntax error'
520
521    def test_python_application_loading_error(self, skip_alert):
522        skip_alert(r'Python failed to import module "blah"')
523
524        self.load('empty', module="blah")
525
526        assert self.get()['status'] == 503, 'loading error'
527
528    def test_python_application_close(self):
529        self.load('close')
530
531        self.get()
532
533        assert self.wait_for_record(r'Close called\.') is not None, 'close'
534
535    def test_python_application_close_error(self):
536        self.load('close_error')
537
538        self.get()
539
540        assert (
541            self.wait_for_record(r'Close called\.') is not None
542        ), 'close error'
543
544    def test_python_application_not_iterable(self):
545        self.load('not_iterable')
546
547        self.get()
548
549        assert (
550            self.wait_for_record(
551                r'\[error\].+the application returned not an iterable object'
552            )
553            is not None
554        ), 'not iterable'
555
556    def test_python_application_write(self):
557        self.load('write')
558
559        assert self.get()['body'] == '0123456789', 'write'
560
561    def test_python_application_threading(self):
562        """wait_for_record() timeouts after 5s while every thread works at
563        least 3s.  So without releasing GIL test should fail.
564        """
565
566        self.load('threading')
567
568        for _ in range(10):
569            self.get(no_recv=True)
570
571        assert (
572            self.wait_for_record(r'\(5\) Thread: 100', wait=50) is not None
573        ), 'last thread finished'
574
575    def test_python_application_iter_exception(self):
576        self.load('iter_exception')
577
578        # Default request doesn't lead to the exception.
579
580        resp = self.get(
581            headers={
582                'Host': 'localhost',
583                'X-Skip': '9',
584                'X-Chunked': '1',
585                'Connection': 'close',
586            }
587        )
588        assert resp['status'] == 200, 'status'
589        assert resp['body'] == 'XXXXXXX', 'body'
590
591        # Exception before start_response().
592
593        assert self.get()['status'] == 503, 'error'
594
595        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
596        assert (
597            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
598            is not None
599        ), 'first exception raise'
600        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
601
602        # Exception after start_response(), before first write().
603
604        assert (
605            self.get(
606                headers={
607                    'Host': 'localhost',
608                    'X-Skip': '1',
609                    'Connection': 'close',
610                }
611            )['status']
612            == 503
613        ), 'error 2'
614
615        assert (
616            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
617            is not None
618        ), 'exception raise second'
619        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
620
621        # Exception after first write(), before first __next__().
622
623        _, sock = self.get(
624            headers={
625                'Host': 'localhost',
626                'X-Skip': '2',
627                'Connection': 'keep-alive',
628            },
629            start=True,
630        )
631
632        assert (
633            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
634            is not None
635        ), 'exception raise third'
636        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
637
638        assert self.get(sock=sock) == {}, 'closed connection'
639
640        # Exception after first write(), before first __next__(),
641        # chunked (incomplete body).
642
643        resp = self.get(
644            headers={
645                'Host': 'localhost',
646                'X-Skip': '2',
647                'X-Chunked': '1',
648                'Connection': 'close',
649            },
650            raw_resp=True,
651        )
652        if resp:
653            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
654        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
655
656        # Exception in __next__().
657
658        _, sock = self.get(
659            headers={
660                'Host': 'localhost',
661                'X-Skip': '3',
662                'Connection': 'keep-alive',
663            },
664            start=True,
665        )
666
667        assert (
668            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
669            is not None
670        ), 'exception raise next'
671        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
672
673        assert self.get(sock=sock) == {}, 'closed connection 2'
674
675        # Exception in __next__(), chunked (incomplete body).
676
677        resp = self.get(
678            headers={
679                'Host': 'localhost',
680                'X-Skip': '3',
681                'X-Chunked': '1',
682                'Connection': 'close',
683            },
684            raw_resp=True,
685        )
686        if resp:
687            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
688        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
689
690        # Exception before start_response() and in close().
691
692        assert (
693            self.get(
694                headers={
695                    'Host': 'localhost',
696                    'X-Not-Skip-Close': '1',
697                    'Connection': 'close',
698                }
699            )['status']
700            == 503
701        ), 'error'
702
703        assert (
704            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
705            is not None
706        ), 'exception raise close'
707        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
708
709    def test_python_user_group(self, is_su):
710        if not is_su:
711            pytest.skip('requires root')
712
713        nobody_uid = pwd.getpwnam('nobody').pw_uid
714
715        group = 'nobody'
716
717        try:
718            group_id = grp.getgrnam(group).gr_gid
719        except KeyError:
720            group = 'nogroup'
721            group_id = grp.getgrnam(group).gr_gid
722
723        self.load('user_group')
724
725        obj = self.getjson()['body']
726        assert obj['UID'] == nobody_uid, 'nobody uid'
727        assert obj['GID'] == group_id, 'nobody gid'
728
729        self.load('user_group', user='nobody')
730
731        obj = self.getjson()['body']
732        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
733        assert obj['GID'] == group_id, 'nobody gid user=nobody'
734
735        self.load('user_group', user='nobody', group=group)
736
737        obj = self.getjson()['body']
738        assert obj['UID'] == nobody_uid, (
739            'nobody uid user=nobody group=%s' % group
740        )
741
742        assert obj['GID'] == group_id, (
743            'nobody gid user=nobody group=%s' % group
744        )
745
746        self.load('user_group', group=group)
747
748        obj = self.getjson()['body']
749        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
750
751        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
752
753        self.load('user_group', user='root')
754
755        obj = self.getjson()['body']
756        assert obj['UID'] == 0, 'root uid user=root'
757        assert obj['GID'] == 0, 'root gid user=root'
758
759        group = 'root'
760
761        try:
762            grp.getgrnam(group)
763            group = True
764        except KeyError:
765            group = False
766
767        if group:
768            self.load('user_group', user='root', group='root')
769
770            obj = self.getjson()['body']
771            assert obj['UID'] == 0, 'root uid user=root group=root'
772            assert obj['GID'] == 0, 'root gid user=root group=root'
773
774            self.load('user_group', group='root')
775
776            obj = self.getjson()['body']
777            assert obj['UID'] == nobody_uid, 'root uid group=root'
778            assert obj['GID'] == 0, 'root gid group=root'
779
780    def test_python_application_callable(self, skip_alert):
781        skip_alert(r'Python failed to get "blah" from module')
782        self.load('callable')
783
784        assert self.get()['status'] == 204, 'default application response'
785
786        self.load('callable', callable="app")
787
788        assert self.get()['status'] == 200, 'callable response'
789
790        self.load('callable', callable="blah")
791
792        assert self.get()['status'] not in [200, 204], 'callable response inv'
793
794    def test_python_application_path(self):
795        self.load('path')
796
797        def set_path(path):
798            assert 'success' in self.conf(path, 'applications/path/path')
799
800        def get_path():
801            return self.get()['body'].split(os.pathsep)
802
803        default_path = self.conf_get('/config/applications/path/path')
804        assert 'success' in self.conf(
805            {"PYTHONPATH": default_path},
806            '/config/applications/path/environment',
807        )
808
809        self.conf_delete('/config/applications/path/path')
810        sys_path = get_path()
811
812        set_path('"/blah"')
813        assert ['/blah', *sys_path] == get_path(), 'check path'
814
815        set_path('"/new"')
816        assert ['/new', *sys_path] == get_path(), 'check path update'
817
818        set_path('["/blah1", "/blah2"]')
819        assert ['/blah1', '/blah2', *sys_path] == get_path(), 'check path array'
820
821    def test_python_application_path_invalid(self):
822        self.load('path')
823
824        def check_path(path):
825            assert 'error' in self.conf(path, 'applications/path/path')
826
827        check_path('{}')
828        check_path('["/blah", []]')
829
830    def test_python_application_threads(self):
831        self.load('threads', threads=4)
832
833        socks = []
834
835        for i in range(4):
836            (_, sock) = self.get(
837                headers={
838                    'Host': 'localhost',
839                    'X-Delay': '2',
840                    'Connection': 'close',
841                },
842                no_recv=True,
843                start=True,
844            )
845
846            socks.append(sock)
847
848        threads = set()
849
850        for sock in socks:
851            resp = self.recvall(sock).decode('utf-8')
852
853            self.log_in(resp)
854
855            resp = self._resp_to_dict(resp)
856
857            assert resp['status'] == 200, 'status'
858
859            threads.add(resp['headers']['X-Thread'])
860
861            assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread'
862
863            sock.close()
864
865        assert len(socks) == len(threads), 'threads differs'
866