xref: /unit/test/test_python_application.py (revision 1604:8cd5ee21340a)
1import grp
2import pytest
3import pwd
4import re
5import time
6
7from unit.applications.lang.python import TestApplicationPython
8from conftest import skip_alert
9
10
11class TestPythonApplication(TestApplicationPython):
12    prerequisites = {'modules': {'python': 'all'}}
13
14    def findall(self, pattern):
15        with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f:
16            return re.findall(pattern, f.read())
17
18    def test_python_application_variables(self):
19        self.load('variables')
20
21        body = 'Test body string.'
22
23        resp = self.post(
24            headers={
25                'Host': 'localhost',
26                'Content-Type': 'text/html',
27                'Custom-Header': 'blah',
28                'Connection': 'close',
29            },
30            body=body,
31        )
32
33        assert resp['status'] == 200, 'status'
34        headers = resp['headers']
35        header_server = headers.pop('Server')
36        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
37        assert (
38            headers.pop('Server-Software') == header_server
39        ), 'server software header'
40
41        date = headers.pop('Date')
42        assert date[-4:] == ' GMT', 'date header timezone'
43        assert (
44            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
45        ), 'date header'
46
47        assert headers == {
48            'Connection': 'close',
49            'Content-Length': str(len(body)),
50            'Content-Type': 'text/html',
51            'Request-Method': 'POST',
52            'Request-Uri': '/',
53            'Http-Host': 'localhost',
54            'Server-Protocol': 'HTTP/1.1',
55            'Custom-Header': 'blah',
56            'Wsgi-Version': '(1, 0)',
57            'Wsgi-Url-Scheme': 'http',
58            'Wsgi-Multithread': 'False',
59            'Wsgi-Multiprocess': 'True',
60            'Wsgi-Run-Once': 'False',
61        }, 'headers'
62        assert resp['body'] == body, 'body'
63
64    def test_python_application_query_string(self):
65        self.load('query_string')
66
67        resp = self.get(url='/?var1=val1&var2=val2')
68
69        assert (
70            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
71        ), 'Query-String header'
72
73    def test_python_application_query_string_space(self):
74        self.load('query_string')
75
76        resp = self.get(url='/ ?var1=val1&var2=val2')
77        assert (
78            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
79        ), 'Query-String space'
80
81        resp = self.get(url='/ %20?var1=val1&var2=val2')
82        assert (
83            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
84        ), 'Query-String space 2'
85
86        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
87        assert (
88            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
89        ), 'Query-String space 3'
90
91        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
92        assert (
93            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
94        ), 'Query-String space 4'
95
96    def test_python_application_query_string_empty(self):
97        self.load('query_string')
98
99        resp = self.get(url='/?')
100
101        assert resp['status'] == 200, 'query string empty status'
102        assert resp['headers']['Query-String'] == '', 'query string empty'
103
104    def test_python_application_query_string_absent(self):
105        self.load('query_string')
106
107        resp = self.get()
108
109        assert resp['status'] == 200, 'query string absent status'
110        assert resp['headers']['Query-String'] == '', 'query string absent'
111
112    @pytest.mark.skip('not yet')
113    def test_python_application_server_port(self):
114        self.load('server_port')
115
116        assert (
117            self.get()['headers']['Server-Port'] == '7080'
118        ), 'Server-Port header'
119
120    @pytest.mark.skip('not yet')
121    def test_python_application_working_directory_invalid(self):
122        self.load('empty')
123
124        assert 'success' in self.conf(
125            '"/blah"', 'applications/empty/working_directory'
126        ), 'configure invalid working_directory'
127
128        assert self.get()['status'] == 500, 'status'
129
130    def test_python_application_204_transfer_encoding(self):
131        self.load('204_no_content')
132
133        assert (
134            'Transfer-Encoding' not in self.get()['headers']
135        ), '204 header transfer encoding'
136
137    def test_python_application_ctx_iter_atexit(self):
138        self.load('ctx_iter_atexit')
139
140        resp = self.post(
141            headers={
142                'Host': 'localhost',
143                'Connection': 'close',
144                'Content-Type': 'text/html',
145            },
146            body='0123456789',
147        )
148
149        assert resp['status'] == 200, 'ctx iter status'
150        assert resp['body'] == '0123456789', 'ctx iter body'
151
152        self.conf({"listeners": {}, "applications": {}})
153
154        self.stop()
155
156        assert (
157            self.wait_for_record(r'RuntimeError') is not None
158        ), 'ctx iter atexit'
159
160    def test_python_keepalive_body(self):
161        self.load('mirror')
162
163        assert self.get()['status'] == 200, 'init'
164
165        body = '0123456789' * 500
166        (resp, sock) = self.post(
167            headers={
168                'Host': 'localhost',
169                'Connection': 'keep-alive',
170                'Content-Type': 'text/html',
171            },
172            start=True,
173            body=body,
174            read_timeout=1,
175        )
176
177        assert resp['body'] == body, 'keep-alive 1'
178
179        body = '0123456789'
180        resp = self.post(
181            headers={
182                'Host': 'localhost',
183                'Connection': 'close',
184                'Content-Type': 'text/html',
185            },
186            sock=sock,
187            body=body,
188        )
189
190        assert resp['body'] == body, 'keep-alive 2'
191
192    def test_python_keepalive_reconfigure(self):
193        skip_alert(
194            r'pthread_mutex.+failed',
195            r'failed to apply',
196            r'process \d+ exited on signal',
197        )
198        self.load('mirror')
199
200        assert self.get()['status'] == 200, 'init'
201
202        body = '0123456789'
203        conns = 3
204        socks = []
205
206        for i in range(conns):
207            (resp, sock) = self.post(
208                headers={
209                    'Host': 'localhost',
210                    'Connection': 'keep-alive',
211                    'Content-Type': 'text/html',
212                },
213                start=True,
214                body=body,
215                read_timeout=1,
216            )
217
218            assert resp['body'] == body, 'keep-alive open'
219            assert 'success' in self.conf(
220                str(i + 1), 'applications/mirror/processes'
221            ), 'reconfigure'
222
223            socks.append(sock)
224
225        for i in range(conns):
226            (resp, sock) = self.post(
227                headers={
228                    'Host': 'localhost',
229                    'Connection': 'keep-alive',
230                    'Content-Type': 'text/html',
231                },
232                start=True,
233                sock=socks[i],
234                body=body,
235                read_timeout=1,
236            )
237
238            assert resp['body'] == body, 'keep-alive request'
239            assert 'success' in self.conf(
240                str(i + 1), 'applications/mirror/processes'
241            ), 'reconfigure 2'
242
243        for i in range(conns):
244            resp = self.post(
245                headers={
246                    'Host': 'localhost',
247                    'Connection': 'close',
248                    'Content-Type': 'text/html',
249                },
250                sock=socks[i],
251                body=body,
252            )
253
254            assert resp['body'] == body, 'keep-alive close'
255            assert 'success' in self.conf(
256                str(i + 1), 'applications/mirror/processes'
257            ), 'reconfigure 3'
258
259    def test_python_keepalive_reconfigure_2(self):
260        self.load('mirror')
261
262        assert self.get()['status'] == 200, 'init'
263
264        body = '0123456789'
265
266        (resp, sock) = self.post(
267            headers={
268                'Host': 'localhost',
269                'Connection': 'keep-alive',
270                'Content-Type': 'text/html',
271            },
272            start=True,
273            body=body,
274            read_timeout=1,
275        )
276
277        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
278
279        self.load('empty')
280
281        assert self.get()['status'] == 200, 'init'
282
283        (resp, sock) = self.post(
284            headers={
285                'Host': 'localhost',
286                'Connection': 'close',
287                'Content-Type': 'text/html',
288            },
289            start=True,
290            sock=sock,
291            body=body,
292        )
293
294        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
295        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
296
297        assert 'success' in self.conf(
298            {"listeners": {}, "applications": {}}
299        ), 'reconfigure 2 clear configuration'
300
301        resp = self.get(sock=sock)
302
303        assert resp == {}, 'reconfigure 2 keep-alive 3'
304
305    def test_python_keepalive_reconfigure_3(self):
306        self.load('empty')
307
308        assert self.get()['status'] == 200, 'init'
309
310        (_, sock) = self.http(
311            b"""GET / HTTP/1.1
312""",
313            start=True,
314            raw=True,
315            no_recv=True,
316        )
317
318        assert self.get()['status'] == 200
319
320        assert 'success' in self.conf(
321            {"listeners": {}, "applications": {}}
322        ), 'reconfigure 3 clear configuration'
323
324        resp = self.http(
325            b"""Host: localhost
326Connection: close
327
328""",
329            sock=sock,
330            raw=True,
331        )
332
333        assert resp['status'] == 200, 'reconfigure 3'
334
335    def test_python_atexit(self):
336        self.load('atexit')
337
338        self.get()
339
340        self.conf({"listeners": {}, "applications": {}})
341
342        self.stop()
343
344        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
345
346    def test_python_process_switch(self):
347        self.load('delayed')
348
349        assert 'success' in self.conf(
350            '2', 'applications/delayed/processes'
351        ), 'configure 2 processes'
352
353        self.get(
354            headers={
355                'Host': 'localhost',
356                'Content-Length': '0',
357                'X-Delay': '5',
358                'Connection': 'close',
359            },
360            no_recv=True,
361        )
362
363        headers_delay_1 = {
364            'Connection': 'close',
365            'Host': 'localhost',
366            'Content-Length': '0',
367            'X-Delay': '1',
368        }
369
370        self.get(headers=headers_delay_1, no_recv=True)
371
372        time.sleep(0.5)
373
374        for _ in range(10):
375            self.get(headers=headers_delay_1, no_recv=True)
376
377        self.get(headers=headers_delay_1)
378
379    @pytest.mark.skip('not yet')
380    def test_python_application_start_response_exit(self):
381        self.load('start_response_exit')
382
383        assert self.get()['status'] == 500, 'start response exit'
384
385    def test_python_application_input_iter(self):
386        self.load('input_iter')
387
388        body = '''0123456789
389next line
390
391last line'''
392
393        resp = self.post(body=body)
394        assert resp['body'] == body, 'input iter'
395        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
396
397    def test_python_application_input_readline(self):
398        self.load('input_readline')
399
400        body = '''0123456789
401next line
402
403last line'''
404
405        resp = self.post(body=body)
406        assert resp['body'] == body, 'input readline'
407        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
408
409    def test_python_application_input_readline_size(self):
410        self.load('input_readline_size')
411
412        body = '''0123456789
413next line
414
415last line'''
416
417        assert self.post(body=body)['body'] == body, 'input readline size'
418        assert (
419            self.post(body='0123')['body'] == '0123'
420        ), 'input readline size less'
421
422    def test_python_application_input_readlines(self):
423        self.load('input_readlines')
424
425        body = '''0123456789
426next line
427
428last line'''
429
430        resp = self.post(body=body)
431        assert resp['body'] == body, 'input readlines'
432        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
433
434    def test_python_application_input_readlines_huge(self):
435        self.load('input_readlines')
436
437        body = (
438            '''0123456789 abcdefghi
439next line: 0123456789 abcdefghi
440
441last line: 987654321
442'''
443            * 512
444        )
445
446        assert (
447            self.post(body=body, read_buffer_size=16384)['body'] == body
448        ), 'input readlines huge'
449
450    def test_python_application_input_read_length(self):
451        self.load('input_read_length')
452
453        body = '0123456789'
454
455        resp = self.post(
456            headers={
457                'Host': 'localhost',
458                'Input-Length': '5',
459                'Connection': 'close',
460            },
461            body=body,
462        )
463
464        assert resp['body'] == body[:5], 'input read length lt body'
465
466        resp = self.post(
467            headers={
468                'Host': 'localhost',
469                'Input-Length': '15',
470                'Connection': 'close',
471            },
472            body=body,
473        )
474
475        assert resp['body'] == body, 'input read length gt body'
476
477        resp = self.post(
478            headers={
479                'Host': 'localhost',
480                'Input-Length': '0',
481                'Connection': 'close',
482            },
483            body=body,
484        )
485
486        assert resp['body'] == '', 'input read length zero'
487
488        resp = self.post(
489            headers={
490                'Host': 'localhost',
491                'Input-Length': '-1',
492                'Connection': 'close',
493            },
494            body=body,
495        )
496
497        assert resp['body'] == body, 'input read length negative'
498
499    @pytest.mark.skip('not yet')
500    def test_python_application_errors_write(self):
501        self.load('errors_write')
502
503        self.get()
504
505        self.stop()
506
507        assert (
508            self.wait_for_record(r'\[error\].+Error in application\.')
509            is not None
510        ), 'errors write'
511
512    def test_python_application_body_array(self):
513        self.load('body_array')
514
515        assert self.get()['body'] == '0123456789', 'body array'
516
517    def test_python_application_body_io(self):
518        self.load('body_io')
519
520        assert self.get()['body'] == '0123456789', 'body io'
521
522    def test_python_application_body_io_file(self):
523        self.load('body_io_file')
524
525        assert self.get()['body'] == 'body\n', 'body io file'
526
527    @pytest.mark.skip('not yet')
528    def test_python_application_syntax_error(self):
529        skip_alert(r'Python failed to import module "wsgi"')
530        self.load('syntax_error')
531
532        assert self.get()['status'] == 500, 'syntax error'
533
534    def test_python_application_loading_error(self):
535        skip_alert(r'Python failed to import module "blah"')
536
537        self.load('empty')
538
539        assert 'success' in self.conf('"blah"', 'applications/empty/module')
540
541        assert self.get()['status'] == 503, 'loading error'
542
543    def test_python_application_close(self):
544        self.load('close')
545
546        self.get()
547
548        self.stop()
549
550        assert self.wait_for_record(r'Close called\.') is not None, 'close'
551
552    def test_python_application_close_error(self):
553        self.load('close_error')
554
555        self.get()
556
557        self.stop()
558
559        assert (
560            self.wait_for_record(r'Close called\.') is not None
561        ), 'close error'
562
563    def test_python_application_not_iterable(self):
564        self.load('not_iterable')
565
566        self.get()
567
568        self.stop()
569
570        assert (
571            self.wait_for_record(
572                r'\[error\].+the application returned not an iterable object'
573            )
574            is not None
575        ), 'not iterable'
576
577    def test_python_application_write(self):
578        self.load('write')
579
580        assert self.get()['body'] == '0123456789', 'write'
581
582    def test_python_application_threading(self):
583        """wait_for_record() timeouts after 5s while every thread works at
584        least 3s.  So without releasing GIL test should fail.
585        """
586
587        self.load('threading')
588
589        for _ in range(10):
590            self.get(no_recv=True)
591
592        assert (
593            self.wait_for_record(r'\(5\) Thread: 100') is not None
594        ), 'last thread finished'
595
596    def test_python_application_iter_exception(self):
597        self.load('iter_exception')
598
599        # Default request doesn't lead to the exception.
600
601        resp = self.get(
602            headers={
603                'Host': 'localhost',
604                'X-Skip': '9',
605                'X-Chunked': '1',
606                'Connection': 'close',
607            }
608        )
609        assert resp['status'] == 200, 'status'
610        assert resp['body'] == 'XXXXXXX', 'body'
611
612        # Exception before start_response().
613
614        assert self.get()['status'] == 503, 'error'
615
616        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
617        assert (
618            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
619            is not None
620        ), 'first exception raise'
621        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
622
623        # Exception after start_response(), before first write().
624
625        assert (
626            self.get(
627                headers={
628                    'Host': 'localhost',
629                    'X-Skip': '1',
630                    'Connection': 'close',
631                }
632            )['status']
633            == 503
634        ), 'error 2'
635
636        assert (
637            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
638            is not None
639        ), 'exception raise second'
640        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
641
642        # Exception after first write(), before first __next__().
643
644        _, sock = self.get(
645            headers={
646                'Host': 'localhost',
647                'X-Skip': '2',
648                'Connection': 'keep-alive',
649            },
650            start=True,
651        )
652
653        assert (
654            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
655            is not None
656        ), 'exception raise third'
657        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
658
659        assert self.get(sock=sock) == {}, 'closed connection'
660
661        # Exception after first write(), before first __next__(),
662        # chunked (incomplete body).
663
664        resp = self.get(
665            headers={
666                'Host': 'localhost',
667                'X-Skip': '2',
668                'X-Chunked': '1',
669                'Connection': 'close',
670            },
671            raw_resp=True,
672        )
673        if resp:
674            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
675        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
676
677        # Exception in __next__().
678
679        _, sock = self.get(
680            headers={
681                'Host': 'localhost',
682                'X-Skip': '3',
683                'Connection': 'keep-alive',
684            },
685            start=True,
686        )
687
688        assert (
689            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
690            is not None
691        ), 'exception raise next'
692        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
693
694        assert self.get(sock=sock) == {}, 'closed connection 2'
695
696        # Exception in __next__(), chunked (incomplete body).
697
698        resp = self.get(
699            headers={
700                'Host': 'localhost',
701                'X-Skip': '3',
702                'X-Chunked': '1',
703                'Connection': 'close',
704            },
705            raw_resp=True,
706        )
707        if resp:
708            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
709        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
710
711        # Exception before start_response() and in close().
712
713        assert (
714            self.get(
715                headers={
716                    'Host': 'localhost',
717                    'X-Not-Skip-Close': '1',
718                    'Connection': 'close',
719                }
720            )['status']
721            == 503
722        ), 'error'
723
724        assert (
725            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
726            is not None
727        ), 'exception raise close'
728        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
729
730    def test_python_user_group(self, is_su):
731        if not is_su:
732            pytest.skip('requires root')
733
734        nobody_uid = pwd.getpwnam('nobody').pw_uid
735
736        group = 'nobody'
737
738        try:
739            group_id = grp.getgrnam(group).gr_gid
740        except:
741            group = 'nogroup'
742            group_id = grp.getgrnam(group).gr_gid
743
744        self.load('user_group')
745
746        obj = self.getjson()['body']
747        assert obj['UID'] == nobody_uid, 'nobody uid'
748        assert obj['GID'] == group_id, 'nobody gid'
749
750        self.load('user_group', user='nobody')
751
752        obj = self.getjson()['body']
753        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
754        assert obj['GID'] == group_id, 'nobody gid user=nobody'
755
756        self.load('user_group', user='nobody', group=group)
757
758        obj = self.getjson()['body']
759        assert obj['UID'] == nobody_uid, (
760            'nobody uid user=nobody group=%s' % group
761        )
762
763        assert obj['GID'] == group_id, (
764            'nobody gid user=nobody group=%s' % group
765        )
766
767        self.load('user_group', group=group)
768
769        obj = self.getjson()['body']
770        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
771
772        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
773
774        self.load('user_group', user='root')
775
776        obj = self.getjson()['body']
777        assert obj['UID'] == 0, 'root uid user=root'
778        assert obj['GID'] == 0, 'root gid user=root'
779
780        group = 'root'
781
782        try:
783            grp.getgrnam(group)
784            group = True
785        except:
786            group = False
787
788        if group:
789            self.load('user_group', user='root', group='root')
790
791            obj = self.getjson()['body']
792            assert obj['UID'] == 0, 'root uid user=root group=root'
793            assert obj['GID'] == 0, 'root gid user=root group=root'
794
795            self.load('user_group', group='root')
796
797            obj = self.getjson()['body']
798            assert obj['UID'] == nobody_uid, 'root uid group=root'
799            assert obj['GID'] == 0, 'root gid group=root'
800
801    def test_python_application_callable(self):
802        skip_alert(r'Python failed to get "blah" from module')
803        self.load('callable')
804
805        assert self.get()['status'] == 204, 'default application response'
806
807        assert 'success' in self.conf(
808            '"app"', 'applications/callable/callable'
809        )
810
811        assert self.get()['status'] == 200, 'callable response'
812
813        assert 'success' in self.conf(
814            '"blah"', 'applications/callable/callable'
815        )
816
817        assert self.get()['status'] not in [200, 204], 'callable response inv'
818
819        assert 'success' in self.conf(
820            '"app"', 'applications/callable/callable'
821        )
822
823        assert self.get()['status'] == 200, 'callable response 2'
824
825        assert 'success' in self.conf_delete('applications/callable/callable')
826
827        assert self.get()['status'] == 204, 'default response 2'
828