xref: /unit/test/test_python_application.py (revision 1771:94cf6c5fafbd)
1import grp
2import pwd
3import re
4import time
5
6import pytest
7from unit.applications.lang.python import TestApplicationPython
8from unit.option import option
9
10
11class TestPythonApplication(TestApplicationPython):
12    prerequisites = {'modules': {'python': 'all'}}
13
14    def findall(self, pattern):
15        with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f:
16            return re.findall(pattern, f.read())
17
18    def test_python_application_variables(self):
19        self.load('variables')
20
21        body = 'Test body string.'
22
23        resp = self.http(
24            b"""POST / HTTP/1.1
25Host: localhost
26Content-Length: %d
27Custom-Header: blah
28Custom-hEader: Blah
29Content-Type: text/html
30Connection: close
31custom-header: BLAH
32
33%s""" % (len(body), body.encode()),
34            raw=True,
35        )
36
37        assert resp['status'] == 200, 'status'
38        headers = resp['headers']
39        header_server = headers.pop('Server')
40        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
41        assert (
42            headers.pop('Server-Software') == header_server
43        ), 'server software header'
44
45        date = headers.pop('Date')
46        assert date[-4:] == ' GMT', 'date header timezone'
47        assert (
48            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
49        ), 'date header'
50
51        assert headers == {
52            'Connection': 'close',
53            'Content-Length': str(len(body)),
54            'Content-Type': 'text/html',
55            'Request-Method': 'POST',
56            'Request-Uri': '/',
57            'Http-Host': 'localhost',
58            'Server-Protocol': 'HTTP/1.1',
59            'Custom-Header': 'blah, Blah, BLAH',
60            'Wsgi-Version': '(1, 0)',
61            'Wsgi-Url-Scheme': 'http',
62            'Wsgi-Multithread': 'False',
63            'Wsgi-Multiprocess': 'True',
64            'Wsgi-Run-Once': 'False',
65        }, 'headers'
66        assert resp['body'] == body, 'body'
67
68    def test_python_application_query_string(self):
69        self.load('query_string')
70
71        resp = self.get(url='/?var1=val1&var2=val2')
72
73        assert (
74            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
75        ), 'Query-String header'
76
77    def test_python_application_query_string_space(self):
78        self.load('query_string')
79
80        resp = self.get(url='/ ?var1=val1&var2=val2')
81        assert (
82            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
83        ), 'Query-String space'
84
85        resp = self.get(url='/ %20?var1=val1&var2=val2')
86        assert (
87            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
88        ), 'Query-String space 2'
89
90        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
91        assert (
92            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
93        ), 'Query-String space 3'
94
95        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
96        assert (
97            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
98        ), 'Query-String space 4'
99
100    def test_python_application_query_string_empty(self):
101        self.load('query_string')
102
103        resp = self.get(url='/?')
104
105        assert resp['status'] == 200, 'query string empty status'
106        assert resp['headers']['Query-String'] == '', 'query string empty'
107
108    def test_python_application_query_string_absent(self):
109        self.load('query_string')
110
111        resp = self.get()
112
113        assert resp['status'] == 200, 'query string absent status'
114        assert resp['headers']['Query-String'] == '', 'query string absent'
115
116    @pytest.mark.skip('not yet')
117    def test_python_application_server_port(self):
118        self.load('server_port')
119
120        assert (
121            self.get()['headers']['Server-Port'] == '7080'
122        ), 'Server-Port header'
123
124    @pytest.mark.skip('not yet')
125    def test_python_application_working_directory_invalid(self):
126        self.load('empty')
127
128        assert 'success' in self.conf(
129            '"/blah"', 'applications/empty/working_directory'
130        ), 'configure invalid working_directory'
131
132        assert self.get()['status'] == 500, 'status'
133
134    def test_python_application_204_transfer_encoding(self):
135        self.load('204_no_content')
136
137        assert (
138            'Transfer-Encoding' not in self.get()['headers']
139        ), '204 header transfer encoding'
140
141    def test_python_application_ctx_iter_atexit(self):
142        self.load('ctx_iter_atexit')
143
144        resp = self.post(
145            headers={
146                'Host': 'localhost',
147                'Connection': 'close',
148                'Content-Type': 'text/html',
149            },
150            body='0123456789',
151        )
152
153        assert resp['status'] == 200, 'ctx iter status'
154        assert resp['body'] == '0123456789', 'ctx iter body'
155
156        self.conf({"listeners": {}, "applications": {}})
157
158        assert (
159            self.wait_for_record(r'RuntimeError') is not None
160        ), 'ctx iter atexit'
161
162    def test_python_keepalive_body(self):
163        self.load('mirror')
164
165        assert self.get()['status'] == 200, 'init'
166
167        body = '0123456789' * 500
168        (resp, sock) = self.post(
169            headers={
170                'Host': 'localhost',
171                'Connection': 'keep-alive',
172                'Content-Type': 'text/html',
173            },
174            start=True,
175            body=body,
176            read_timeout=1,
177        )
178
179        assert resp['body'] == body, 'keep-alive 1'
180
181        body = '0123456789'
182        resp = self.post(
183            headers={
184                'Host': 'localhost',
185                'Connection': 'close',
186                'Content-Type': 'text/html',
187            },
188            sock=sock,
189            body=body,
190        )
191
192        assert resp['body'] == body, 'keep-alive 2'
193
194    def test_python_keepalive_reconfigure(self):
195        self.load('mirror')
196
197        assert self.get()['status'] == 200, 'init'
198
199        body = '0123456789'
200        conns = 3
201        socks = []
202
203        for i in range(conns):
204            (resp, sock) = self.post(
205                headers={
206                    'Host': 'localhost',
207                    'Connection': 'keep-alive',
208                    'Content-Type': 'text/html',
209                },
210                start=True,
211                body=body,
212                read_timeout=1,
213            )
214
215            assert resp['body'] == body, 'keep-alive open'
216
217            self.load('mirror', processes=i + 1)
218
219            socks.append(sock)
220
221        for i in range(conns):
222            (resp, sock) = self.post(
223                headers={
224                    'Host': 'localhost',
225                    'Connection': 'keep-alive',
226                    'Content-Type': 'text/html',
227                },
228                start=True,
229                sock=socks[i],
230                body=body,
231                read_timeout=1,
232            )
233
234            assert resp['body'] == body, 'keep-alive request'
235
236            self.load('mirror', processes=i + 1)
237
238        for i in range(conns):
239            resp = self.post(
240                headers={
241                    'Host': 'localhost',
242                    'Connection': 'close',
243                    'Content-Type': 'text/html',
244                },
245                sock=socks[i],
246                body=body,
247            )
248
249            assert resp['body'] == body, 'keep-alive close'
250
251            self.load('mirror', processes=i + 1)
252
253    def test_python_keepalive_reconfigure_2(self):
254        self.load('mirror')
255
256        assert self.get()['status'] == 200, 'init'
257
258        body = '0123456789'
259
260        (resp, sock) = self.post(
261            headers={
262                'Host': 'localhost',
263                'Connection': 'keep-alive',
264                'Content-Type': 'text/html',
265            },
266            start=True,
267            body=body,
268            read_timeout=1,
269        )
270
271        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
272
273        self.load('empty')
274
275        assert self.get()['status'] == 200, 'init'
276
277        (resp, sock) = self.post(
278            headers={
279                'Host': 'localhost',
280                'Connection': 'close',
281                'Content-Type': 'text/html',
282            },
283            start=True,
284            sock=sock,
285            body=body,
286        )
287
288        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
289        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
290
291        assert 'success' in self.conf(
292            {"listeners": {}, "applications": {}}
293        ), 'reconfigure 2 clear configuration'
294
295        resp = self.get(sock=sock)
296
297        assert resp == {}, 'reconfigure 2 keep-alive 3'
298
299    def test_python_keepalive_reconfigure_3(self):
300        self.load('empty')
301
302        assert self.get()['status'] == 200, 'init'
303
304        (_, sock) = self.http(
305            b"""GET / HTTP/1.1
306""",
307            start=True,
308            raw=True,
309            no_recv=True,
310        )
311
312        assert self.get()['status'] == 200
313
314        assert 'success' in self.conf(
315            {"listeners": {}, "applications": {}}
316        ), 'reconfigure 3 clear configuration'
317
318        resp = self.http(
319            b"""Host: localhost
320Connection: close
321
322""",
323            sock=sock,
324            raw=True,
325        )
326
327        assert resp['status'] == 200, 'reconfigure 3'
328
329    def test_python_atexit(self):
330        self.load('atexit')
331
332        self.get()
333
334        self.conf({"listeners": {}, "applications": {}})
335
336        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
337
338    def test_python_process_switch(self):
339        self.load('delayed', processes=2)
340
341        self.get(
342            headers={
343                'Host': 'localhost',
344                'Content-Length': '0',
345                'X-Delay': '5',
346                'Connection': 'close',
347            },
348            no_recv=True,
349        )
350
351        headers_delay_1 = {
352            'Connection': 'close',
353            'Host': 'localhost',
354            'Content-Length': '0',
355            'X-Delay': '1',
356        }
357
358        self.get(headers=headers_delay_1, no_recv=True)
359
360        time.sleep(0.5)
361
362        for _ in range(10):
363            self.get(headers=headers_delay_1, no_recv=True)
364
365        self.get(headers=headers_delay_1)
366
367    @pytest.mark.skip('not yet')
368    def test_python_application_start_response_exit(self):
369        self.load('start_response_exit')
370
371        assert self.get()['status'] == 500, 'start response exit'
372
373    def test_python_application_input_iter(self):
374        self.load('input_iter')
375
376        body = '''0123456789
377next line
378
379last line'''
380
381        resp = self.post(body=body)
382        assert resp['body'] == body, 'input iter'
383        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
384
385    def test_python_application_input_readline(self):
386        self.load('input_readline')
387
388        body = '''0123456789
389next line
390
391last line'''
392
393        resp = self.post(body=body)
394        assert resp['body'] == body, 'input readline'
395        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
396
397    def test_python_application_input_readline_size(self):
398        self.load('input_readline_size')
399
400        body = '''0123456789
401next line
402
403last line'''
404
405        assert self.post(body=body)['body'] == body, 'input readline size'
406        assert (
407            self.post(body='0123')['body'] == '0123'
408        ), 'input readline size less'
409
410    def test_python_application_input_readlines(self):
411        self.load('input_readlines')
412
413        body = '''0123456789
414next line
415
416last line'''
417
418        resp = self.post(body=body)
419        assert resp['body'] == body, 'input readlines'
420        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
421
422    def test_python_application_input_readlines_huge(self):
423        self.load('input_readlines')
424
425        body = (
426            '''0123456789 abcdefghi
427next line: 0123456789 abcdefghi
428
429last line: 987654321
430'''
431            * 512
432        )
433
434        assert (
435            self.post(body=body, read_buffer_size=16384)['body'] == body
436        ), 'input readlines huge'
437
438    def test_python_application_input_read_length(self):
439        self.load('input_read_length')
440
441        body = '0123456789'
442
443        resp = self.post(
444            headers={
445                'Host': 'localhost',
446                'Input-Length': '5',
447                'Connection': 'close',
448            },
449            body=body,
450        )
451
452        assert resp['body'] == body[:5], 'input read length lt body'
453
454        resp = self.post(
455            headers={
456                'Host': 'localhost',
457                'Input-Length': '15',
458                'Connection': 'close',
459            },
460            body=body,
461        )
462
463        assert resp['body'] == body, 'input read length gt body'
464
465        resp = self.post(
466            headers={
467                'Host': 'localhost',
468                'Input-Length': '0',
469                'Connection': 'close',
470            },
471            body=body,
472        )
473
474        assert resp['body'] == '', 'input read length zero'
475
476        resp = self.post(
477            headers={
478                'Host': 'localhost',
479                'Input-Length': '-1',
480                'Connection': 'close',
481            },
482            body=body,
483        )
484
485        assert resp['body'] == body, 'input read length negative'
486
487    @pytest.mark.skip('not yet')
488    def test_python_application_errors_write(self):
489        self.load('errors_write')
490
491        self.get()
492
493        assert (
494            self.wait_for_record(r'\[error\].+Error in application\.')
495            is not None
496        ), 'errors write'
497
498    def test_python_application_body_array(self):
499        self.load('body_array')
500
501        assert self.get()['body'] == '0123456789', 'body array'
502
503    def test_python_application_body_io(self):
504        self.load('body_io')
505
506        assert self.get()['body'] == '0123456789', 'body io'
507
508    def test_python_application_body_io_file(self):
509        self.load('body_io_file')
510
511        assert self.get()['body'] == 'body\n', 'body io file'
512
513    @pytest.mark.skip('not yet')
514    def test_python_application_syntax_error(self, skip_alert):
515        skip_alert(r'Python failed to import module "wsgi"')
516        self.load('syntax_error')
517
518        assert self.get()['status'] == 500, 'syntax error'
519
520    def test_python_application_loading_error(self, skip_alert):
521        skip_alert(r'Python failed to import module "blah"')
522
523        self.load('empty', module="blah")
524
525        assert self.get()['status'] == 503, 'loading error'
526
527    def test_python_application_close(self):
528        self.load('close')
529
530        self.get()
531
532        assert self.wait_for_record(r'Close called\.') is not None, 'close'
533
534    def test_python_application_close_error(self):
535        self.load('close_error')
536
537        self.get()
538
539        assert (
540            self.wait_for_record(r'Close called\.') is not None
541        ), 'close error'
542
543    def test_python_application_not_iterable(self):
544        self.load('not_iterable')
545
546        self.get()
547
548        assert (
549            self.wait_for_record(
550                r'\[error\].+the application returned not an iterable object'
551            )
552            is not None
553        ), 'not iterable'
554
555    def test_python_application_write(self):
556        self.load('write')
557
558        assert self.get()['body'] == '0123456789', 'write'
559
560    def test_python_application_threading(self):
561        """wait_for_record() timeouts after 5s while every thread works at
562        least 3s.  So without releasing GIL test should fail.
563        """
564
565        self.load('threading')
566
567        for _ in range(10):
568            self.get(no_recv=True)
569
570        assert (
571            self.wait_for_record(r'\(5\) Thread: 100') is not None
572        ), 'last thread finished'
573
574    def test_python_application_iter_exception(self):
575        self.load('iter_exception')
576
577        # Default request doesn't lead to the exception.
578
579        resp = self.get(
580            headers={
581                'Host': 'localhost',
582                'X-Skip': '9',
583                'X-Chunked': '1',
584                'Connection': 'close',
585            }
586        )
587        assert resp['status'] == 200, 'status'
588        assert resp['body'] == 'XXXXXXX', 'body'
589
590        # Exception before start_response().
591
592        assert self.get()['status'] == 503, 'error'
593
594        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
595        assert (
596            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
597            is not None
598        ), 'first exception raise'
599        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
600
601        # Exception after start_response(), before first write().
602
603        assert (
604            self.get(
605                headers={
606                    'Host': 'localhost',
607                    'X-Skip': '1',
608                    'Connection': 'close',
609                }
610            )['status']
611            == 503
612        ), 'error 2'
613
614        assert (
615            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
616            is not None
617        ), 'exception raise second'
618        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
619
620        # Exception after first write(), before first __next__().
621
622        _, sock = self.get(
623            headers={
624                'Host': 'localhost',
625                'X-Skip': '2',
626                'Connection': 'keep-alive',
627            },
628            start=True,
629        )
630
631        assert (
632            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
633            is not None
634        ), 'exception raise third'
635        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
636
637        assert self.get(sock=sock) == {}, 'closed connection'
638
639        # Exception after first write(), before first __next__(),
640        # chunked (incomplete body).
641
642        resp = self.get(
643            headers={
644                'Host': 'localhost',
645                'X-Skip': '2',
646                'X-Chunked': '1',
647                'Connection': 'close',
648            },
649            raw_resp=True,
650        )
651        if resp:
652            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
653        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
654
655        # Exception in __next__().
656
657        _, sock = self.get(
658            headers={
659                'Host': 'localhost',
660                'X-Skip': '3',
661                'Connection': 'keep-alive',
662            },
663            start=True,
664        )
665
666        assert (
667            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
668            is not None
669        ), 'exception raise next'
670        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
671
672        assert self.get(sock=sock) == {}, 'closed connection 2'
673
674        # Exception in __next__(), chunked (incomplete body).
675
676        resp = self.get(
677            headers={
678                'Host': 'localhost',
679                'X-Skip': '3',
680                'X-Chunked': '1',
681                'Connection': 'close',
682            },
683            raw_resp=True,
684        )
685        if resp:
686            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
687        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
688
689        # Exception before start_response() and in close().
690
691        assert (
692            self.get(
693                headers={
694                    'Host': 'localhost',
695                    'X-Not-Skip-Close': '1',
696                    'Connection': 'close',
697                }
698            )['status']
699            == 503
700        ), 'error'
701
702        assert (
703            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
704            is not None
705        ), 'exception raise close'
706        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
707
708    def test_python_user_group(self, is_su):
709        if not is_su:
710            pytest.skip('requires root')
711
712        nobody_uid = pwd.getpwnam('nobody').pw_uid
713
714        group = 'nobody'
715
716        try:
717            group_id = grp.getgrnam(group).gr_gid
718        except KeyError:
719            group = 'nogroup'
720            group_id = grp.getgrnam(group).gr_gid
721
722        self.load('user_group')
723
724        obj = self.getjson()['body']
725        assert obj['UID'] == nobody_uid, 'nobody uid'
726        assert obj['GID'] == group_id, 'nobody gid'
727
728        self.load('user_group', user='nobody')
729
730        obj = self.getjson()['body']
731        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
732        assert obj['GID'] == group_id, 'nobody gid user=nobody'
733
734        self.load('user_group', user='nobody', group=group)
735
736        obj = self.getjson()['body']
737        assert obj['UID'] == nobody_uid, (
738            'nobody uid user=nobody group=%s' % group
739        )
740
741        assert obj['GID'] == group_id, (
742            'nobody gid user=nobody group=%s' % group
743        )
744
745        self.load('user_group', group=group)
746
747        obj = self.getjson()['body']
748        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
749
750        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
751
752        self.load('user_group', user='root')
753
754        obj = self.getjson()['body']
755        assert obj['UID'] == 0, 'root uid user=root'
756        assert obj['GID'] == 0, 'root gid user=root'
757
758        group = 'root'
759
760        try:
761            grp.getgrnam(group)
762            group = True
763        except KeyError:
764            group = False
765
766        if group:
767            self.load('user_group', user='root', group='root')
768
769            obj = self.getjson()['body']
770            assert obj['UID'] == 0, 'root uid user=root group=root'
771            assert obj['GID'] == 0, 'root gid user=root group=root'
772
773            self.load('user_group', group='root')
774
775            obj = self.getjson()['body']
776            assert obj['UID'] == nobody_uid, 'root uid group=root'
777            assert obj['GID'] == 0, 'root gid group=root'
778
779    def test_python_application_callable(self, skip_alert):
780        skip_alert(r'Python failed to get "blah" from module')
781        self.load('callable')
782
783        assert self.get()['status'] == 204, 'default application response'
784
785        self.load('callable', callable="app")
786
787        assert self.get()['status'] == 200, 'callable response'
788
789        self.load('callable', callable="blah")
790
791        assert self.get()['status'] not in [200, 204], 'callable response inv'
792
793    def test_python_application_threads(self):
794        self.load('threads', threads=4)
795
796        socks = []
797
798        for i in range(4):
799            (_, sock) = self.get(
800                headers={
801                    'Host': 'localhost',
802                    'X-Delay': '2',
803                    'Connection': 'close',
804                },
805                no_recv=True,
806                start=True,
807            )
808
809            socks.append(sock)
810
811        threads = set()
812
813        for sock in socks:
814            resp = self.recvall(sock).decode('utf-8')
815
816            self.log_in(resp)
817
818            resp = self._resp_to_dict(resp)
819
820            assert resp['status'] == 200, 'status'
821
822            threads.add(resp['headers']['X-Thread'])
823
824            assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread'
825
826            sock.close()
827
828        assert len(socks) == len(threads), 'threads differs'
829