xref: /unit/test/test_python_application.py (revision 1736:22db875fda34) 
1import grp
2import pwd
3import re
4import time
5
6import pytest
7
8from conftest import unit_stop
9from unit.applications.lang.python import TestApplicationPython
10from unit.option import option
11
12
13class TestPythonApplication(TestApplicationPython):
14    prerequisites = {'modules': {'python': 'all'}}
15
16    def findall(self, pattern):
17        with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f:
18            return re.findall(pattern, f.read())
19
20    def test_python_application_variables(self):
21        self.load('variables')
22
23        body = 'Test body string.'
24
25        resp = self.http(
26            b"""POST / HTTP/1.1
27Host: localhost
28Content-Length: %d
29Custom-Header: blah
30Custom-hEader: Blah
31Content-Type: text/html
32Connection: close
33custom-header: BLAH
34
35%s""" % (len(body), body.encode()),
36            raw=True,
37        )
38
39        assert resp['status'] == 200, 'status'
40        headers = resp['headers']
41        header_server = headers.pop('Server')
42        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
43        assert (
44            headers.pop('Server-Software') == header_server
45        ), 'server software header'
46
47        date = headers.pop('Date')
48        assert date[-4:] == ' GMT', 'date header timezone'
49        assert (
50            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
51        ), 'date header'
52
53        assert headers == {
54            'Connection': 'close',
55            'Content-Length': str(len(body)),
56            'Content-Type': 'text/html',
57            'Request-Method': 'POST',
58            'Request-Uri': '/',
59            'Http-Host': 'localhost',
60            'Server-Protocol': 'HTTP/1.1',
61            'Custom-Header': 'blah, Blah, BLAH',
62            'Wsgi-Version': '(1, 0)',
63            'Wsgi-Url-Scheme': 'http',
64            'Wsgi-Multithread': 'False',
65            'Wsgi-Multiprocess': 'True',
66            'Wsgi-Run-Once': 'False',
67        }, 'headers'
68        assert resp['body'] == body, 'body'
69
70    def test_python_application_query_string(self):
71        self.load('query_string')
72
73        resp = self.get(url='/?var1=val1&var2=val2')
74
75        assert (
76            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
77        ), 'Query-String header'
78
79    def test_python_application_query_string_space(self):
80        self.load('query_string')
81
82        resp = self.get(url='/ ?var1=val1&var2=val2')
83        assert (
84            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
85        ), 'Query-String space'
86
87        resp = self.get(url='/ %20?var1=val1&var2=val2')
88        assert (
89            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
90        ), 'Query-String space 2'
91
92        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
93        assert (
94            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
95        ), 'Query-String space 3'
96
97        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
98        assert (
99            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
100        ), 'Query-String space 4'
101
102    def test_python_application_query_string_empty(self):
103        self.load('query_string')
104
105        resp = self.get(url='/?')
106
107        assert resp['status'] == 200, 'query string empty status'
108        assert resp['headers']['Query-String'] == '', 'query string empty'
109
110    def test_python_application_query_string_absent(self):
111        self.load('query_string')
112
113        resp = self.get()
114
115        assert resp['status'] == 200, 'query string absent status'
116        assert resp['headers']['Query-String'] == '', 'query string absent'
117
118    @pytest.mark.skip('not yet')
119    def test_python_application_server_port(self):
120        self.load('server_port')
121
122        assert (
123            self.get()['headers']['Server-Port'] == '7080'
124        ), 'Server-Port header'
125
126    @pytest.mark.skip('not yet')
127    def test_python_application_working_directory_invalid(self):
128        self.load('empty')
129
130        assert 'success' in self.conf(
131            '"/blah"', 'applications/empty/working_directory'
132        ), 'configure invalid working_directory'
133
134        assert self.get()['status'] == 500, 'status'
135
136    def test_python_application_204_transfer_encoding(self):
137        self.load('204_no_content')
138
139        assert (
140            'Transfer-Encoding' not in self.get()['headers']
141        ), '204 header transfer encoding'
142
143    def test_python_application_ctx_iter_atexit(self):
144        self.load('ctx_iter_atexit')
145
146        resp = self.post(
147            headers={
148                'Host': 'localhost',
149                'Connection': 'close',
150                'Content-Type': 'text/html',
151            },
152            body='0123456789',
153        )
154
155        assert resp['status'] == 200, 'ctx iter status'
156        assert resp['body'] == '0123456789', 'ctx iter body'
157
158        self.conf({"listeners": {}, "applications": {}})
159
160        unit_stop()
161
162        assert (
163            self.wait_for_record(r'RuntimeError') is not None
164        ), 'ctx iter atexit'
165
166    def test_python_keepalive_body(self):
167        self.load('mirror')
168
169        assert self.get()['status'] == 200, 'init'
170
171        body = '0123456789' * 500
172        (resp, sock) = self.post(
173            headers={
174                'Host': 'localhost',
175                'Connection': 'keep-alive',
176                'Content-Type': 'text/html',
177            },
178            start=True,
179            body=body,
180            read_timeout=1,
181        )
182
183        assert resp['body'] == body, 'keep-alive 1'
184
185        body = '0123456789'
186        resp = self.post(
187            headers={
188                'Host': 'localhost',
189                'Connection': 'close',
190                'Content-Type': 'text/html',
191            },
192            sock=sock,
193            body=body,
194        )
195
196        assert resp['body'] == body, 'keep-alive 2'
197
198    def test_python_keepalive_reconfigure(self):
199        self.load('mirror')
200
201        assert self.get()['status'] == 200, 'init'
202
203        body = '0123456789'
204        conns = 3
205        socks = []
206
207        for i in range(conns):
208            (resp, sock) = self.post(
209                headers={
210                    'Host': 'localhost',
211                    'Connection': 'keep-alive',
212                    'Content-Type': 'text/html',
213                },
214                start=True,
215                body=body,
216                read_timeout=1,
217            )
218
219            assert resp['body'] == body, 'keep-alive open'
220
221            self.load('mirror', processes=i + 1)
222
223            socks.append(sock)
224
225        for i in range(conns):
226            (resp, sock) = self.post(
227                headers={
228                    'Host': 'localhost',
229                    'Connection': 'keep-alive',
230                    'Content-Type': 'text/html',
231                },
232                start=True,
233                sock=socks[i],
234                body=body,
235                read_timeout=1,
236            )
237
238            assert resp['body'] == body, 'keep-alive request'
239
240            self.load('mirror', processes=i + 1)
241
242        for i in range(conns):
243            resp = self.post(
244                headers={
245                    'Host': 'localhost',
246                    'Connection': 'close',
247                    'Content-Type': 'text/html',
248                },
249                sock=socks[i],
250                body=body,
251            )
252
253            assert resp['body'] == body, 'keep-alive close'
254
255            self.load('mirror', processes=i + 1)
256
257    def test_python_keepalive_reconfigure_2(self):
258        self.load('mirror')
259
260        assert self.get()['status'] == 200, 'init'
261
262        body = '0123456789'
263
264        (resp, sock) = self.post(
265            headers={
266                'Host': 'localhost',
267                'Connection': 'keep-alive',
268                'Content-Type': 'text/html',
269            },
270            start=True,
271            body=body,
272            read_timeout=1,
273        )
274
275        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
276
277        self.load('empty')
278
279        assert self.get()['status'] == 200, 'init'
280
281        (resp, sock) = self.post(
282            headers={
283                'Host': 'localhost',
284                'Connection': 'close',
285                'Content-Type': 'text/html',
286            },
287            start=True,
288            sock=sock,
289            body=body,
290        )
291
292        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
293        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
294
295        assert 'success' in self.conf(
296            {"listeners": {}, "applications": {}}
297        ), 'reconfigure 2 clear configuration'
298
299        resp = self.get(sock=sock)
300
301        assert resp == {}, 'reconfigure 2 keep-alive 3'
302
303    def test_python_keepalive_reconfigure_3(self):
304        self.load('empty')
305
306        assert self.get()['status'] == 200, 'init'
307
308        (_, sock) = self.http(
309            b"""GET / HTTP/1.1
310""",
311            start=True,
312            raw=True,
313            no_recv=True,
314        )
315
316        assert self.get()['status'] == 200
317
318        assert 'success' in self.conf(
319            {"listeners": {}, "applications": {}}
320        ), 'reconfigure 3 clear configuration'
321
322        resp = self.http(
323            b"""Host: localhost
324Connection: close
325
326""",
327            sock=sock,
328            raw=True,
329        )
330
331        assert resp['status'] == 200, 'reconfigure 3'
332
333    def test_python_atexit(self):
334        self.load('atexit')
335
336        self.get()
337
338        self.conf({"listeners": {}, "applications": {}})
339
340        unit_stop()
341
342        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
343
344    def test_python_process_switch(self):
345        self.load('delayed', processes=2)
346
347        self.get(
348            headers={
349                'Host': 'localhost',
350                'Content-Length': '0',
351                'X-Delay': '5',
352                'Connection': 'close',
353            },
354            no_recv=True,
355        )
356
357        headers_delay_1 = {
358            'Connection': 'close',
359            'Host': 'localhost',
360            'Content-Length': '0',
361            'X-Delay': '1',
362        }
363
364        self.get(headers=headers_delay_1, no_recv=True)
365
366        time.sleep(0.5)
367
368        for _ in range(10):
369            self.get(headers=headers_delay_1, no_recv=True)
370
371        self.get(headers=headers_delay_1)
372
373    @pytest.mark.skip('not yet')
374    def test_python_application_start_response_exit(self):
375        self.load('start_response_exit')
376
377        assert self.get()['status'] == 500, 'start response exit'
378
379    def test_python_application_input_iter(self):
380        self.load('input_iter')
381
382        body = '''0123456789
383next line
384
385last line'''
386
387        resp = self.post(body=body)
388        assert resp['body'] == body, 'input iter'
389        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
390
391    def test_python_application_input_readline(self):
392        self.load('input_readline')
393
394        body = '''0123456789
395next line
396
397last line'''
398
399        resp = self.post(body=body)
400        assert resp['body'] == body, 'input readline'
401        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
402
403    def test_python_application_input_readline_size(self):
404        self.load('input_readline_size')
405
406        body = '''0123456789
407next line
408
409last line'''
410
411        assert self.post(body=body)['body'] == body, 'input readline size'
412        assert (
413            self.post(body='0123')['body'] == '0123'
414        ), 'input readline size less'
415
416    def test_python_application_input_readlines(self):
417        self.load('input_readlines')
418
419        body = '''0123456789
420next line
421
422last line'''
423
424        resp = self.post(body=body)
425        assert resp['body'] == body, 'input readlines'
426        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
427
428    def test_python_application_input_readlines_huge(self):
429        self.load('input_readlines')
430
431        body = (
432            '''0123456789 abcdefghi
433next line: 0123456789 abcdefghi
434
435last line: 987654321
436'''
437            * 512
438        )
439
440        assert (
441            self.post(body=body, read_buffer_size=16384)['body'] == body
442        ), 'input readlines huge'
443
444    def test_python_application_input_read_length(self):
445        self.load('input_read_length')
446
447        body = '0123456789'
448
449        resp = self.post(
450            headers={
451                'Host': 'localhost',
452                'Input-Length': '5',
453                'Connection': 'close',
454            },
455            body=body,
456        )
457
458        assert resp['body'] == body[:5], 'input read length lt body'
459
460        resp = self.post(
461            headers={
462                'Host': 'localhost',
463                'Input-Length': '15',
464                'Connection': 'close',
465            },
466            body=body,
467        )
468
469        assert resp['body'] == body, 'input read length gt body'
470
471        resp = self.post(
472            headers={
473                'Host': 'localhost',
474                'Input-Length': '0',
475                'Connection': 'close',
476            },
477            body=body,
478        )
479
480        assert resp['body'] == '', 'input read length zero'
481
482        resp = self.post(
483            headers={
484                'Host': 'localhost',
485                'Input-Length': '-1',
486                'Connection': 'close',
487            },
488            body=body,
489        )
490
491        assert resp['body'] == body, 'input read length negative'
492
493    @pytest.mark.skip('not yet')
494    def test_python_application_errors_write(self):
495        self.load('errors_write')
496
497        self.get()
498
499        unit_stop()
500
501        assert (
502            self.wait_for_record(r'\[error\].+Error in application\.')
503            is not None
504        ), 'errors write'
505
506    def test_python_application_body_array(self):
507        self.load('body_array')
508
509        assert self.get()['body'] == '0123456789', 'body array'
510
511    def test_python_application_body_io(self):
512        self.load('body_io')
513
514        assert self.get()['body'] == '0123456789', 'body io'
515
516    def test_python_application_body_io_file(self):
517        self.load('body_io_file')
518
519        assert self.get()['body'] == 'body\n', 'body io file'
520
521    @pytest.mark.skip('not yet')
522    def test_python_application_syntax_error(self, skip_alert):
523        skip_alert(r'Python failed to import module "wsgi"')
524        self.load('syntax_error')
525
526        assert self.get()['status'] == 500, 'syntax error'
527
528    def test_python_application_loading_error(self, skip_alert):
529        skip_alert(r'Python failed to import module "blah"')
530
531        self.load('empty', module="blah")
532
533        assert self.get()['status'] == 503, 'loading error'
534
535    def test_python_application_close(self):
536        self.load('close')
537
538        self.get()
539
540        unit_stop()
541
542        assert self.wait_for_record(r'Close called\.') is not None, 'close'
543
544    def test_python_application_close_error(self):
545        self.load('close_error')
546
547        self.get()
548
549        unit_stop()
550
551        assert (
552            self.wait_for_record(r'Close called\.') is not None
553        ), 'close error'
554
555    def test_python_application_not_iterable(self):
556        self.load('not_iterable')
557
558        self.get()
559
560        unit_stop()
561
562        assert (
563            self.wait_for_record(
564                r'\[error\].+the application returned not an iterable object'
565            )
566            is not None
567        ), 'not iterable'
568
569    def test_python_application_write(self):
570        self.load('write')
571
572        assert self.get()['body'] == '0123456789', 'write'
573
574    def test_python_application_threading(self):
575        """wait_for_record() timeouts after 5s while every thread works at
576        least 3s.  So without releasing GIL test should fail.
577        """
578
579        self.load('threading')
580
581        for _ in range(10):
582            self.get(no_recv=True)
583
584        assert (
585            self.wait_for_record(r'\(5\) Thread: 100') is not None
586        ), 'last thread finished'
587
588    def test_python_application_iter_exception(self):
589        self.load('iter_exception')
590
591        # Default request doesn't lead to the exception.
592
593        resp = self.get(
594            headers={
595                'Host': 'localhost',
596                'X-Skip': '9',
597                'X-Chunked': '1',
598                'Connection': 'close',
599            }
600        )
601        assert resp['status'] == 200, 'status'
602        assert resp['body'] == 'XXXXXXX', 'body'
603
604        # Exception before start_response().
605
606        assert self.get()['status'] == 503, 'error'
607
608        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
609        assert (
610            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
611            is not None
612        ), 'first exception raise'
613        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
614
615        # Exception after start_response(), before first write().
616
617        assert (
618            self.get(
619                headers={
620                    'Host': 'localhost',
621                    'X-Skip': '1',
622                    'Connection': 'close',
623                }
624            )['status']
625            == 503
626        ), 'error 2'
627
628        assert (
629            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
630            is not None
631        ), 'exception raise second'
632        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
633
634        # Exception after first write(), before first __next__().
635
636        _, sock = self.get(
637            headers={
638                'Host': 'localhost',
639                'X-Skip': '2',
640                'Connection': 'keep-alive',
641            },
642            start=True,
643        )
644
645        assert (
646            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
647            is not None
648        ), 'exception raise third'
649        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
650
651        assert self.get(sock=sock) == {}, 'closed connection'
652
653        # Exception after first write(), before first __next__(),
654        # chunked (incomplete body).
655
656        resp = self.get(
657            headers={
658                'Host': 'localhost',
659                'X-Skip': '2',
660                'X-Chunked': '1',
661                'Connection': 'close',
662            },
663            raw_resp=True,
664        )
665        if resp:
666            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
667        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
668
669        # Exception in __next__().
670
671        _, sock = self.get(
672            headers={
673                'Host': 'localhost',
674                'X-Skip': '3',
675                'Connection': 'keep-alive',
676            },
677            start=True,
678        )
679
680        assert (
681            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
682            is not None
683        ), 'exception raise next'
684        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
685
686        assert self.get(sock=sock) == {}, 'closed connection 2'
687
688        # Exception in __next__(), chunked (incomplete body).
689
690        resp = self.get(
691            headers={
692                'Host': 'localhost',
693                'X-Skip': '3',
694                'X-Chunked': '1',
695                'Connection': 'close',
696            },
697            raw_resp=True,
698        )
699        if resp:
700            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
701        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
702
703        # Exception before start_response() and in close().
704
705        assert (
706            self.get(
707                headers={
708                    'Host': 'localhost',
709                    'X-Not-Skip-Close': '1',
710                    'Connection': 'close',
711                }
712            )['status']
713            == 503
714        ), 'error'
715
716        assert (
717            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
718            is not None
719        ), 'exception raise close'
720        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
721
722    def test_python_user_group(self, is_su):
723        if not is_su:
724            pytest.skip('requires root')
725
726        nobody_uid = pwd.getpwnam('nobody').pw_uid
727
728        group = 'nobody'
729
730        try:
731            group_id = grp.getgrnam(group).gr_gid
732        except KeyError:
733            group = 'nogroup'
734            group_id = grp.getgrnam(group).gr_gid
735
736        self.load('user_group')
737
738        obj = self.getjson()['body']
739        assert obj['UID'] == nobody_uid, 'nobody uid'
740        assert obj['GID'] == group_id, 'nobody gid'
741
742        self.load('user_group', user='nobody')
743
744        obj = self.getjson()['body']
745        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
746        assert obj['GID'] == group_id, 'nobody gid user=nobody'
747
748        self.load('user_group', user='nobody', group=group)
749
750        obj = self.getjson()['body']
751        assert obj['UID'] == nobody_uid, (
752            'nobody uid user=nobody group=%s' % group
753        )
754
755        assert obj['GID'] == group_id, (
756            'nobody gid user=nobody group=%s' % group
757        )
758
759        self.load('user_group', group=group)
760
761        obj = self.getjson()['body']
762        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
763
764        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
765
766        self.load('user_group', user='root')
767
768        obj = self.getjson()['body']
769        assert obj['UID'] == 0, 'root uid user=root'
770        assert obj['GID'] == 0, 'root gid user=root'
771
772        group = 'root'
773
774        try:
775            grp.getgrnam(group)
776            group = True
777        except KeyError:
778            group = False
779
780        if group:
781            self.load('user_group', user='root', group='root')
782
783            obj = self.getjson()['body']
784            assert obj['UID'] == 0, 'root uid user=root group=root'
785            assert obj['GID'] == 0, 'root gid user=root group=root'
786
787            self.load('user_group', group='root')
788
789            obj = self.getjson()['body']
790            assert obj['UID'] == nobody_uid, 'root uid group=root'
791            assert obj['GID'] == 0, 'root gid group=root'
792
793    def test_python_application_callable(self, skip_alert):
794        skip_alert(r'Python failed to get "blah" from module')
795        self.load('callable')
796
797        assert self.get()['status'] == 204, 'default application response'
798
799        self.load('callable', callable="app")
800
801        assert self.get()['status'] == 200, 'callable response'
802
803        self.load('callable', callable="blah")
804
805        assert self.get()['status'] not in [200, 204], 'callable response inv'
806
807    def test_python_application_threads(self):
808        self.load('threads', threads=4)
809
810        socks = []
811
812        for i in range(4):
813            (_, sock) = self.get(
814                headers={
815                    'Host': 'localhost',
816                    'X-Delay': '2',
817                    'Connection': 'close',
818                },
819                no_recv=True,
820                start=True,
821            )
822
823            socks.append(sock)
824
825        threads = set()
826
827        for sock in socks:
828            resp = self.recvall(sock).decode('utf-8')
829
830            self.log_in(resp)
831
832            resp = self._resp_to_dict(resp)
833
834            assert resp['status'] == 200, 'status'
835
836            threads.add(resp['headers']['X-Thread'])
837
838            assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread'
839
840            sock.close()
841
842        assert len(socks) == len(threads), 'threads differs'
843