xref: /unit/test/test_python_application.py (revision 1605:bac76b00ded6)
1import grp
2import pytest
3import pwd
4import re
5import time
6
7from unit.applications.lang.python import TestApplicationPython
8from conftest import skip_alert
9
10
11class TestPythonApplication(TestApplicationPython):
12    prerequisites = {'modules': {'python': 'all'}}
13
14    def findall(self, pattern):
15        with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f:
16            return re.findall(pattern, f.read())
17
18    def test_python_application_variables(self):
19        self.load('variables')
20
21        body = 'Test body string.'
22
23        resp = self.http(
24            b"""POST / HTTP/1.1
25Host: localhost
26Content-Length: %d
27Custom-Header: blah
28Custom-hEader: Blah
29Content-Type: text/html
30Connection: close
31custom-header: BLAH
32
33%s""" % (len(body), body.encode()),
34            raw=True,
35        )
36
37        assert resp['status'] == 200, 'status'
38        headers = resp['headers']
39        header_server = headers.pop('Server')
40        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
41        assert (
42            headers.pop('Server-Software') == header_server
43        ), 'server software header'
44
45        date = headers.pop('Date')
46        assert date[-4:] == ' GMT', 'date header timezone'
47        assert (
48            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
49        ), 'date header'
50
51        assert headers == {
52            'Connection': 'close',
53            'Content-Length': str(len(body)),
54            'Content-Type': 'text/html',
55            'Request-Method': 'POST',
56            'Request-Uri': '/',
57            'Http-Host': 'localhost',
58            'Server-Protocol': 'HTTP/1.1',
59            'Custom-Header': 'blah, Blah, BLAH',
60            'Wsgi-Version': '(1, 0)',
61            'Wsgi-Url-Scheme': 'http',
62            'Wsgi-Multithread': 'False',
63            'Wsgi-Multiprocess': 'True',
64            'Wsgi-Run-Once': 'False',
65        }, 'headers'
66        assert resp['body'] == body, 'body'
67
68    def test_python_application_query_string(self):
69        self.load('query_string')
70
71        resp = self.get(url='/?var1=val1&var2=val2')
72
73        assert (
74            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
75        ), 'Query-String header'
76
77    def test_python_application_query_string_space(self):
78        self.load('query_string')
79
80        resp = self.get(url='/ ?var1=val1&var2=val2')
81        assert (
82            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
83        ), 'Query-String space'
84
85        resp = self.get(url='/ %20?var1=val1&var2=val2')
86        assert (
87            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
88        ), 'Query-String space 2'
89
90        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
91        assert (
92            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
93        ), 'Query-String space 3'
94
95        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
96        assert (
97            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
98        ), 'Query-String space 4'
99
100    def test_python_application_query_string_empty(self):
101        self.load('query_string')
102
103        resp = self.get(url='/?')
104
105        assert resp['status'] == 200, 'query string empty status'
106        assert resp['headers']['Query-String'] == '', 'query string empty'
107
108    def test_python_application_query_string_absent(self):
109        self.load('query_string')
110
111        resp = self.get()
112
113        assert resp['status'] == 200, 'query string absent status'
114        assert resp['headers']['Query-String'] == '', 'query string absent'
115
116    @pytest.mark.skip('not yet')
117    def test_python_application_server_port(self):
118        self.load('server_port')
119
120        assert (
121            self.get()['headers']['Server-Port'] == '7080'
122        ), 'Server-Port header'
123
124    @pytest.mark.skip('not yet')
125    def test_python_application_working_directory_invalid(self):
126        self.load('empty')
127
128        assert 'success' in self.conf(
129            '"/blah"', 'applications/empty/working_directory'
130        ), 'configure invalid working_directory'
131
132        assert self.get()['status'] == 500, 'status'
133
134    def test_python_application_204_transfer_encoding(self):
135        self.load('204_no_content')
136
137        assert (
138            'Transfer-Encoding' not in self.get()['headers']
139        ), '204 header transfer encoding'
140
141    def test_python_application_ctx_iter_atexit(self):
142        self.load('ctx_iter_atexit')
143
144        resp = self.post(
145            headers={
146                'Host': 'localhost',
147                'Connection': 'close',
148                'Content-Type': 'text/html',
149            },
150            body='0123456789',
151        )
152
153        assert resp['status'] == 200, 'ctx iter status'
154        assert resp['body'] == '0123456789', 'ctx iter body'
155
156        self.conf({"listeners": {}, "applications": {}})
157
158        self.stop()
159
160        assert (
161            self.wait_for_record(r'RuntimeError') is not None
162        ), 'ctx iter atexit'
163
164    def test_python_keepalive_body(self):
165        self.load('mirror')
166
167        assert self.get()['status'] == 200, 'init'
168
169        body = '0123456789' * 500
170        (resp, sock) = self.post(
171            headers={
172                'Host': 'localhost',
173                'Connection': 'keep-alive',
174                'Content-Type': 'text/html',
175            },
176            start=True,
177            body=body,
178            read_timeout=1,
179        )
180
181        assert resp['body'] == body, 'keep-alive 1'
182
183        body = '0123456789'
184        resp = self.post(
185            headers={
186                'Host': 'localhost',
187                'Connection': 'close',
188                'Content-Type': 'text/html',
189            },
190            sock=sock,
191            body=body,
192        )
193
194        assert resp['body'] == body, 'keep-alive 2'
195
196    def test_python_keepalive_reconfigure(self):
197        skip_alert(
198            r'pthread_mutex.+failed',
199            r'failed to apply',
200            r'process \d+ exited on signal',
201        )
202        self.load('mirror')
203
204        assert self.get()['status'] == 200, 'init'
205
206        body = '0123456789'
207        conns = 3
208        socks = []
209
210        for i in range(conns):
211            (resp, sock) = self.post(
212                headers={
213                    'Host': 'localhost',
214                    'Connection': 'keep-alive',
215                    'Content-Type': 'text/html',
216                },
217                start=True,
218                body=body,
219                read_timeout=1,
220            )
221
222            assert resp['body'] == body, 'keep-alive open'
223            assert 'success' in self.conf(
224                str(i + 1), 'applications/mirror/processes'
225            ), 'reconfigure'
226
227            socks.append(sock)
228
229        for i in range(conns):
230            (resp, sock) = self.post(
231                headers={
232                    'Host': 'localhost',
233                    'Connection': 'keep-alive',
234                    'Content-Type': 'text/html',
235                },
236                start=True,
237                sock=socks[i],
238                body=body,
239                read_timeout=1,
240            )
241
242            assert resp['body'] == body, 'keep-alive request'
243            assert 'success' in self.conf(
244                str(i + 1), 'applications/mirror/processes'
245            ), 'reconfigure 2'
246
247        for i in range(conns):
248            resp = self.post(
249                headers={
250                    'Host': 'localhost',
251                    'Connection': 'close',
252                    'Content-Type': 'text/html',
253                },
254                sock=socks[i],
255                body=body,
256            )
257
258            assert resp['body'] == body, 'keep-alive close'
259            assert 'success' in self.conf(
260                str(i + 1), 'applications/mirror/processes'
261            ), 'reconfigure 3'
262
263    def test_python_keepalive_reconfigure_2(self):
264        self.load('mirror')
265
266        assert self.get()['status'] == 200, 'init'
267
268        body = '0123456789'
269
270        (resp, sock) = self.post(
271            headers={
272                'Host': 'localhost',
273                'Connection': 'keep-alive',
274                'Content-Type': 'text/html',
275            },
276            start=True,
277            body=body,
278            read_timeout=1,
279        )
280
281        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
282
283        self.load('empty')
284
285        assert self.get()['status'] == 200, 'init'
286
287        (resp, sock) = self.post(
288            headers={
289                'Host': 'localhost',
290                'Connection': 'close',
291                'Content-Type': 'text/html',
292            },
293            start=True,
294            sock=sock,
295            body=body,
296        )
297
298        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
299        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
300
301        assert 'success' in self.conf(
302            {"listeners": {}, "applications": {}}
303        ), 'reconfigure 2 clear configuration'
304
305        resp = self.get(sock=sock)
306
307        assert resp == {}, 'reconfigure 2 keep-alive 3'
308
309    def test_python_keepalive_reconfigure_3(self):
310        self.load('empty')
311
312        assert self.get()['status'] == 200, 'init'
313
314        (_, sock) = self.http(
315            b"""GET / HTTP/1.1
316""",
317            start=True,
318            raw=True,
319            no_recv=True,
320        )
321
322        assert self.get()['status'] == 200
323
324        assert 'success' in self.conf(
325            {"listeners": {}, "applications": {}}
326        ), 'reconfigure 3 clear configuration'
327
328        resp = self.http(
329            b"""Host: localhost
330Connection: close
331
332""",
333            sock=sock,
334            raw=True,
335        )
336
337        assert resp['status'] == 200, 'reconfigure 3'
338
339    def test_python_atexit(self):
340        self.load('atexit')
341
342        self.get()
343
344        self.conf({"listeners": {}, "applications": {}})
345
346        self.stop()
347
348        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
349
350    def test_python_process_switch(self):
351        self.load('delayed')
352
353        assert 'success' in self.conf(
354            '2', 'applications/delayed/processes'
355        ), 'configure 2 processes'
356
357        self.get(
358            headers={
359                'Host': 'localhost',
360                'Content-Length': '0',
361                'X-Delay': '5',
362                'Connection': 'close',
363            },
364            no_recv=True,
365        )
366
367        headers_delay_1 = {
368            'Connection': 'close',
369            'Host': 'localhost',
370            'Content-Length': '0',
371            'X-Delay': '1',
372        }
373
374        self.get(headers=headers_delay_1, no_recv=True)
375
376        time.sleep(0.5)
377
378        for _ in range(10):
379            self.get(headers=headers_delay_1, no_recv=True)
380
381        self.get(headers=headers_delay_1)
382
383    @pytest.mark.skip('not yet')
384    def test_python_application_start_response_exit(self):
385        self.load('start_response_exit')
386
387        assert self.get()['status'] == 500, 'start response exit'
388
389    def test_python_application_input_iter(self):
390        self.load('input_iter')
391
392        body = '''0123456789
393next line
394
395last line'''
396
397        resp = self.post(body=body)
398        assert resp['body'] == body, 'input iter'
399        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
400
401    def test_python_application_input_readline(self):
402        self.load('input_readline')
403
404        body = '''0123456789
405next line
406
407last line'''
408
409        resp = self.post(body=body)
410        assert resp['body'] == body, 'input readline'
411        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
412
413    def test_python_application_input_readline_size(self):
414        self.load('input_readline_size')
415
416        body = '''0123456789
417next line
418
419last line'''
420
421        assert self.post(body=body)['body'] == body, 'input readline size'
422        assert (
423            self.post(body='0123')['body'] == '0123'
424        ), 'input readline size less'
425
426    def test_python_application_input_readlines(self):
427        self.load('input_readlines')
428
429        body = '''0123456789
430next line
431
432last line'''
433
434        resp = self.post(body=body)
435        assert resp['body'] == body, 'input readlines'
436        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
437
438    def test_python_application_input_readlines_huge(self):
439        self.load('input_readlines')
440
441        body = (
442            '''0123456789 abcdefghi
443next line: 0123456789 abcdefghi
444
445last line: 987654321
446'''
447            * 512
448        )
449
450        assert (
451            self.post(body=body, read_buffer_size=16384)['body'] == body
452        ), 'input readlines huge'
453
454    def test_python_application_input_read_length(self):
455        self.load('input_read_length')
456
457        body = '0123456789'
458
459        resp = self.post(
460            headers={
461                'Host': 'localhost',
462                'Input-Length': '5',
463                'Connection': 'close',
464            },
465            body=body,
466        )
467
468        assert resp['body'] == body[:5], 'input read length lt body'
469
470        resp = self.post(
471            headers={
472                'Host': 'localhost',
473                'Input-Length': '15',
474                'Connection': 'close',
475            },
476            body=body,
477        )
478
479        assert resp['body'] == body, 'input read length gt body'
480
481        resp = self.post(
482            headers={
483                'Host': 'localhost',
484                'Input-Length': '0',
485                'Connection': 'close',
486            },
487            body=body,
488        )
489
490        assert resp['body'] == '', 'input read length zero'
491
492        resp = self.post(
493            headers={
494                'Host': 'localhost',
495                'Input-Length': '-1',
496                'Connection': 'close',
497            },
498            body=body,
499        )
500
501        assert resp['body'] == body, 'input read length negative'
502
503    @pytest.mark.skip('not yet')
504    def test_python_application_errors_write(self):
505        self.load('errors_write')
506
507        self.get()
508
509        self.stop()
510
511        assert (
512            self.wait_for_record(r'\[error\].+Error in application\.')
513            is not None
514        ), 'errors write'
515
516    def test_python_application_body_array(self):
517        self.load('body_array')
518
519        assert self.get()['body'] == '0123456789', 'body array'
520
521    def test_python_application_body_io(self):
522        self.load('body_io')
523
524        assert self.get()['body'] == '0123456789', 'body io'
525
526    def test_python_application_body_io_file(self):
527        self.load('body_io_file')
528
529        assert self.get()['body'] == 'body\n', 'body io file'
530
531    @pytest.mark.skip('not yet')
532    def test_python_application_syntax_error(self):
533        skip_alert(r'Python failed to import module "wsgi"')
534        self.load('syntax_error')
535
536        assert self.get()['status'] == 500, 'syntax error'
537
538    def test_python_application_loading_error(self):
539        skip_alert(r'Python failed to import module "blah"')
540
541        self.load('empty')
542
543        assert 'success' in self.conf('"blah"', 'applications/empty/module')
544
545        assert self.get()['status'] == 503, 'loading error'
546
547    def test_python_application_close(self):
548        self.load('close')
549
550        self.get()
551
552        self.stop()
553
554        assert self.wait_for_record(r'Close called\.') is not None, 'close'
555
556    def test_python_application_close_error(self):
557        self.load('close_error')
558
559        self.get()
560
561        self.stop()
562
563        assert (
564            self.wait_for_record(r'Close called\.') is not None
565        ), 'close error'
566
567    def test_python_application_not_iterable(self):
568        self.load('not_iterable')
569
570        self.get()
571
572        self.stop()
573
574        assert (
575            self.wait_for_record(
576                r'\[error\].+the application returned not an iterable object'
577            )
578            is not None
579        ), 'not iterable'
580
581    def test_python_application_write(self):
582        self.load('write')
583
584        assert self.get()['body'] == '0123456789', 'write'
585
586    def test_python_application_threading(self):
587        """wait_for_record() timeouts after 5s while every thread works at
588        least 3s.  So without releasing GIL test should fail.
589        """
590
591        self.load('threading')
592
593        for _ in range(10):
594            self.get(no_recv=True)
595
596        assert (
597            self.wait_for_record(r'\(5\) Thread: 100') is not None
598        ), 'last thread finished'
599
600    def test_python_application_iter_exception(self):
601        self.load('iter_exception')
602
603        # Default request doesn't lead to the exception.
604
605        resp = self.get(
606            headers={
607                'Host': 'localhost',
608                'X-Skip': '9',
609                'X-Chunked': '1',
610                'Connection': 'close',
611            }
612        )
613        assert resp['status'] == 200, 'status'
614        assert resp['body'] == 'XXXXXXX', 'body'
615
616        # Exception before start_response().
617
618        assert self.get()['status'] == 503, 'error'
619
620        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
621        assert (
622            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
623            is not None
624        ), 'first exception raise'
625        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
626
627        # Exception after start_response(), before first write().
628
629        assert (
630            self.get(
631                headers={
632                    'Host': 'localhost',
633                    'X-Skip': '1',
634                    'Connection': 'close',
635                }
636            )['status']
637            == 503
638        ), 'error 2'
639
640        assert (
641            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
642            is not None
643        ), 'exception raise second'
644        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
645
646        # Exception after first write(), before first __next__().
647
648        _, sock = self.get(
649            headers={
650                'Host': 'localhost',
651                'X-Skip': '2',
652                'Connection': 'keep-alive',
653            },
654            start=True,
655        )
656
657        assert (
658            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
659            is not None
660        ), 'exception raise third'
661        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
662
663        assert self.get(sock=sock) == {}, 'closed connection'
664
665        # Exception after first write(), before first __next__(),
666        # chunked (incomplete body).
667
668        resp = self.get(
669            headers={
670                'Host': 'localhost',
671                'X-Skip': '2',
672                'X-Chunked': '1',
673                'Connection': 'close',
674            },
675            raw_resp=True,
676        )
677        if resp:
678            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
679        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
680
681        # Exception in __next__().
682
683        _, sock = self.get(
684            headers={
685                'Host': 'localhost',
686                'X-Skip': '3',
687                'Connection': 'keep-alive',
688            },
689            start=True,
690        )
691
692        assert (
693            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
694            is not None
695        ), 'exception raise next'
696        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
697
698        assert self.get(sock=sock) == {}, 'closed connection 2'
699
700        # Exception in __next__(), chunked (incomplete body).
701
702        resp = self.get(
703            headers={
704                'Host': 'localhost',
705                'X-Skip': '3',
706                'X-Chunked': '1',
707                'Connection': 'close',
708            },
709            raw_resp=True,
710        )
711        if resp:
712            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
713        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
714
715        # Exception before start_response() and in close().
716
717        assert (
718            self.get(
719                headers={
720                    'Host': 'localhost',
721                    'X-Not-Skip-Close': '1',
722                    'Connection': 'close',
723                }
724            )['status']
725            == 503
726        ), 'error'
727
728        assert (
729            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
730            is not None
731        ), 'exception raise close'
732        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
733
734    def test_python_user_group(self, is_su):
735        if not is_su:
736            pytest.skip('requires root')
737
738        nobody_uid = pwd.getpwnam('nobody').pw_uid
739
740        group = 'nobody'
741
742        try:
743            group_id = grp.getgrnam(group).gr_gid
744        except:
745            group = 'nogroup'
746            group_id = grp.getgrnam(group).gr_gid
747
748        self.load('user_group')
749
750        obj = self.getjson()['body']
751        assert obj['UID'] == nobody_uid, 'nobody uid'
752        assert obj['GID'] == group_id, 'nobody gid'
753
754        self.load('user_group', user='nobody')
755
756        obj = self.getjson()['body']
757        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
758        assert obj['GID'] == group_id, 'nobody gid user=nobody'
759
760        self.load('user_group', user='nobody', group=group)
761
762        obj = self.getjson()['body']
763        assert obj['UID'] == nobody_uid, (
764            'nobody uid user=nobody group=%s' % group
765        )
766
767        assert obj['GID'] == group_id, (
768            'nobody gid user=nobody group=%s' % group
769        )
770
771        self.load('user_group', group=group)
772
773        obj = self.getjson()['body']
774        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
775
776        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
777
778        self.load('user_group', user='root')
779
780        obj = self.getjson()['body']
781        assert obj['UID'] == 0, 'root uid user=root'
782        assert obj['GID'] == 0, 'root gid user=root'
783
784        group = 'root'
785
786        try:
787            grp.getgrnam(group)
788            group = True
789        except:
790            group = False
791
792        if group:
793            self.load('user_group', user='root', group='root')
794
795            obj = self.getjson()['body']
796            assert obj['UID'] == 0, 'root uid user=root group=root'
797            assert obj['GID'] == 0, 'root gid user=root group=root'
798
799            self.load('user_group', group='root')
800
801            obj = self.getjson()['body']
802            assert obj['UID'] == nobody_uid, 'root uid group=root'
803            assert obj['GID'] == 0, 'root gid group=root'
804
805    def test_python_application_callable(self):
806        skip_alert(r'Python failed to get "blah" from module')
807        self.load('callable')
808
809        assert self.get()['status'] == 204, 'default application response'
810
811        assert 'success' in self.conf(
812            '"app"', 'applications/callable/callable'
813        )
814
815        assert self.get()['status'] == 200, 'callable response'
816
817        assert 'success' in self.conf(
818            '"blah"', 'applications/callable/callable'
819        )
820
821        assert self.get()['status'] not in [200, 204], 'callable response inv'
822
823        assert 'success' in self.conf(
824            '"app"', 'applications/callable/callable'
825        )
826
827        assert self.get()['status'] == 200, 'callable response 2'
828
829        assert 'success' in self.conf_delete('applications/callable/callable')
830
831        assert self.get()['status'] == 204, 'default response 2'
832