xref: /unit/test/test_python_application.py (revision 1654:fc7d0578e124)
1import grp
2import pwd
3import re
4import time
5
6import pytest
7
8from conftest import option
9from conftest import skip_alert
10from conftest import unit_stop
11from unit.applications.lang.python import TestApplicationPython
12
13
14class TestPythonApplication(TestApplicationPython):
15    prerequisites = {'modules': {'python': 'all'}}
16
17    def findall(self, pattern):
18        with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f:
19            return re.findall(pattern, f.read())
20
21    def test_python_application_variables(self):
22        self.load('variables')
23
24        body = 'Test body string.'
25
26        resp = self.http(
27            b"""POST / HTTP/1.1
28Host: localhost
29Content-Length: %d
30Custom-Header: blah
31Custom-hEader: Blah
32Content-Type: text/html
33Connection: close
34custom-header: BLAH
35
36%s""" % (len(body), body.encode()),
37            raw=True,
38        )
39
40        assert resp['status'] == 200, 'status'
41        headers = resp['headers']
42        header_server = headers.pop('Server')
43        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
44        assert (
45            headers.pop('Server-Software') == header_server
46        ), 'server software header'
47
48        date = headers.pop('Date')
49        assert date[-4:] == ' GMT', 'date header timezone'
50        assert (
51            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
52        ), 'date header'
53
54        assert headers == {
55            'Connection': 'close',
56            'Content-Length': str(len(body)),
57            'Content-Type': 'text/html',
58            'Request-Method': 'POST',
59            'Request-Uri': '/',
60            'Http-Host': 'localhost',
61            'Server-Protocol': 'HTTP/1.1',
62            'Custom-Header': 'blah, Blah, BLAH',
63            'Wsgi-Version': '(1, 0)',
64            'Wsgi-Url-Scheme': 'http',
65            'Wsgi-Multithread': 'False',
66            'Wsgi-Multiprocess': 'True',
67            'Wsgi-Run-Once': 'False',
68        }, 'headers'
69        assert resp['body'] == body, 'body'
70
71    def test_python_application_query_string(self):
72        self.load('query_string')
73
74        resp = self.get(url='/?var1=val1&var2=val2')
75
76        assert (
77            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
78        ), 'Query-String header'
79
80    def test_python_application_query_string_space(self):
81        self.load('query_string')
82
83        resp = self.get(url='/ ?var1=val1&var2=val2')
84        assert (
85            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
86        ), 'Query-String space'
87
88        resp = self.get(url='/ %20?var1=val1&var2=val2')
89        assert (
90            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
91        ), 'Query-String space 2'
92
93        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
94        assert (
95            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
96        ), 'Query-String space 3'
97
98        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
99        assert (
100            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
101        ), 'Query-String space 4'
102
103    def test_python_application_query_string_empty(self):
104        self.load('query_string')
105
106        resp = self.get(url='/?')
107
108        assert resp['status'] == 200, 'query string empty status'
109        assert resp['headers']['Query-String'] == '', 'query string empty'
110
111    def test_python_application_query_string_absent(self):
112        self.load('query_string')
113
114        resp = self.get()
115
116        assert resp['status'] == 200, 'query string absent status'
117        assert resp['headers']['Query-String'] == '', 'query string absent'
118
119    @pytest.mark.skip('not yet')
120    def test_python_application_server_port(self):
121        self.load('server_port')
122
123        assert (
124            self.get()['headers']['Server-Port'] == '7080'
125        ), 'Server-Port header'
126
127    @pytest.mark.skip('not yet')
128    def test_python_application_working_directory_invalid(self):
129        self.load('empty')
130
131        assert 'success' in self.conf(
132            '"/blah"', 'applications/empty/working_directory'
133        ), 'configure invalid working_directory'
134
135        assert self.get()['status'] == 500, 'status'
136
137    def test_python_application_204_transfer_encoding(self):
138        self.load('204_no_content')
139
140        assert (
141            'Transfer-Encoding' not in self.get()['headers']
142        ), '204 header transfer encoding'
143
144    def test_python_application_ctx_iter_atexit(self):
145        self.load('ctx_iter_atexit')
146
147        resp = self.post(
148            headers={
149                'Host': 'localhost',
150                'Connection': 'close',
151                'Content-Type': 'text/html',
152            },
153            body='0123456789',
154        )
155
156        assert resp['status'] == 200, 'ctx iter status'
157        assert resp['body'] == '0123456789', 'ctx iter body'
158
159        self.conf({"listeners": {}, "applications": {}})
160
161        unit_stop()
162
163        assert (
164            self.wait_for_record(r'RuntimeError') is not None
165        ), 'ctx iter atexit'
166
167    def test_python_keepalive_body(self):
168        self.load('mirror')
169
170        assert self.get()['status'] == 200, 'init'
171
172        body = '0123456789' * 500
173        (resp, sock) = self.post(
174            headers={
175                'Host': 'localhost',
176                'Connection': 'keep-alive',
177                'Content-Type': 'text/html',
178            },
179            start=True,
180            body=body,
181            read_timeout=1,
182        )
183
184        assert resp['body'] == body, 'keep-alive 1'
185
186        body = '0123456789'
187        resp = self.post(
188            headers={
189                'Host': 'localhost',
190                'Connection': 'close',
191                'Content-Type': 'text/html',
192            },
193            sock=sock,
194            body=body,
195        )
196
197        assert resp['body'] == body, 'keep-alive 2'
198
199    def test_python_keepalive_reconfigure(self):
200        skip_alert(
201            r'pthread_mutex.+failed',
202            r'failed to apply',
203            r'process \d+ exited on signal',
204        )
205        self.load('mirror')
206
207        assert self.get()['status'] == 200, 'init'
208
209        body = '0123456789'
210        conns = 3
211        socks = []
212
213        for i in range(conns):
214            (resp, sock) = self.post(
215                headers={
216                    'Host': 'localhost',
217                    'Connection': 'keep-alive',
218                    'Content-Type': 'text/html',
219                },
220                start=True,
221                body=body,
222                read_timeout=1,
223            )
224
225            assert resp['body'] == body, 'keep-alive open'
226            assert 'success' in self.conf(
227                str(i + 1), 'applications/mirror/processes'
228            ), 'reconfigure'
229
230            socks.append(sock)
231
232        for i in range(conns):
233            (resp, sock) = self.post(
234                headers={
235                    'Host': 'localhost',
236                    'Connection': 'keep-alive',
237                    'Content-Type': 'text/html',
238                },
239                start=True,
240                sock=socks[i],
241                body=body,
242                read_timeout=1,
243            )
244
245            assert resp['body'] == body, 'keep-alive request'
246            assert 'success' in self.conf(
247                str(i + 1), 'applications/mirror/processes'
248            ), 'reconfigure 2'
249
250        for i in range(conns):
251            resp = self.post(
252                headers={
253                    'Host': 'localhost',
254                    'Connection': 'close',
255                    'Content-Type': 'text/html',
256                },
257                sock=socks[i],
258                body=body,
259            )
260
261            assert resp['body'] == body, 'keep-alive close'
262            assert 'success' in self.conf(
263                str(i + 1), 'applications/mirror/processes'
264            ), 'reconfigure 3'
265
266    def test_python_keepalive_reconfigure_2(self):
267        self.load('mirror')
268
269        assert self.get()['status'] == 200, 'init'
270
271        body = '0123456789'
272
273        (resp, sock) = self.post(
274            headers={
275                'Host': 'localhost',
276                'Connection': 'keep-alive',
277                'Content-Type': 'text/html',
278            },
279            start=True,
280            body=body,
281            read_timeout=1,
282        )
283
284        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
285
286        self.load('empty')
287
288        assert self.get()['status'] == 200, 'init'
289
290        (resp, sock) = self.post(
291            headers={
292                'Host': 'localhost',
293                'Connection': 'close',
294                'Content-Type': 'text/html',
295            },
296            start=True,
297            sock=sock,
298            body=body,
299        )
300
301        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
302        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
303
304        assert 'success' in self.conf(
305            {"listeners": {}, "applications": {}}
306        ), 'reconfigure 2 clear configuration'
307
308        resp = self.get(sock=sock)
309
310        assert resp == {}, 'reconfigure 2 keep-alive 3'
311
312    def test_python_keepalive_reconfigure_3(self):
313        self.load('empty')
314
315        assert self.get()['status'] == 200, 'init'
316
317        (_, sock) = self.http(
318            b"""GET / HTTP/1.1
319""",
320            start=True,
321            raw=True,
322            no_recv=True,
323        )
324
325        assert self.get()['status'] == 200
326
327        assert 'success' in self.conf(
328            {"listeners": {}, "applications": {}}
329        ), 'reconfigure 3 clear configuration'
330
331        resp = self.http(
332            b"""Host: localhost
333Connection: close
334
335""",
336            sock=sock,
337            raw=True,
338        )
339
340        assert resp['status'] == 200, 'reconfigure 3'
341
342    def test_python_atexit(self):
343        self.load('atexit')
344
345        self.get()
346
347        self.conf({"listeners": {}, "applications": {}})
348
349        unit_stop()
350
351        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
352
353    def test_python_process_switch(self):
354        self.load('delayed')
355
356        assert 'success' in self.conf(
357            '2', 'applications/delayed/processes'
358        ), 'configure 2 processes'
359
360        self.get(
361            headers={
362                'Host': 'localhost',
363                'Content-Length': '0',
364                'X-Delay': '5',
365                'Connection': 'close',
366            },
367            no_recv=True,
368        )
369
370        headers_delay_1 = {
371            'Connection': 'close',
372            'Host': 'localhost',
373            'Content-Length': '0',
374            'X-Delay': '1',
375        }
376
377        self.get(headers=headers_delay_1, no_recv=True)
378
379        time.sleep(0.5)
380
381        for _ in range(10):
382            self.get(headers=headers_delay_1, no_recv=True)
383
384        self.get(headers=headers_delay_1)
385
386    @pytest.mark.skip('not yet')
387    def test_python_application_start_response_exit(self):
388        self.load('start_response_exit')
389
390        assert self.get()['status'] == 500, 'start response exit'
391
392    def test_python_application_input_iter(self):
393        self.load('input_iter')
394
395        body = '''0123456789
396next line
397
398last line'''
399
400        resp = self.post(body=body)
401        assert resp['body'] == body, 'input iter'
402        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
403
404    def test_python_application_input_readline(self):
405        self.load('input_readline')
406
407        body = '''0123456789
408next line
409
410last line'''
411
412        resp = self.post(body=body)
413        assert resp['body'] == body, 'input readline'
414        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
415
416    def test_python_application_input_readline_size(self):
417        self.load('input_readline_size')
418
419        body = '''0123456789
420next line
421
422last line'''
423
424        assert self.post(body=body)['body'] == body, 'input readline size'
425        assert (
426            self.post(body='0123')['body'] == '0123'
427        ), 'input readline size less'
428
429    def test_python_application_input_readlines(self):
430        self.load('input_readlines')
431
432        body = '''0123456789
433next line
434
435last line'''
436
437        resp = self.post(body=body)
438        assert resp['body'] == body, 'input readlines'
439        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
440
441    def test_python_application_input_readlines_huge(self):
442        self.load('input_readlines')
443
444        body = (
445            '''0123456789 abcdefghi
446next line: 0123456789 abcdefghi
447
448last line: 987654321
449'''
450            * 512
451        )
452
453        assert (
454            self.post(body=body, read_buffer_size=16384)['body'] == body
455        ), 'input readlines huge'
456
457    def test_python_application_input_read_length(self):
458        self.load('input_read_length')
459
460        body = '0123456789'
461
462        resp = self.post(
463            headers={
464                'Host': 'localhost',
465                'Input-Length': '5',
466                'Connection': 'close',
467            },
468            body=body,
469        )
470
471        assert resp['body'] == body[:5], 'input read length lt body'
472
473        resp = self.post(
474            headers={
475                'Host': 'localhost',
476                'Input-Length': '15',
477                'Connection': 'close',
478            },
479            body=body,
480        )
481
482        assert resp['body'] == body, 'input read length gt body'
483
484        resp = self.post(
485            headers={
486                'Host': 'localhost',
487                'Input-Length': '0',
488                'Connection': 'close',
489            },
490            body=body,
491        )
492
493        assert resp['body'] == '', 'input read length zero'
494
495        resp = self.post(
496            headers={
497                'Host': 'localhost',
498                'Input-Length': '-1',
499                'Connection': 'close',
500            },
501            body=body,
502        )
503
504        assert resp['body'] == body, 'input read length negative'
505
506    @pytest.mark.skip('not yet')
507    def test_python_application_errors_write(self):
508        self.load('errors_write')
509
510        self.get()
511
512        unit_stop()
513
514        assert (
515            self.wait_for_record(r'\[error\].+Error in application\.')
516            is not None
517        ), 'errors write'
518
519    def test_python_application_body_array(self):
520        self.load('body_array')
521
522        assert self.get()['body'] == '0123456789', 'body array'
523
524    def test_python_application_body_io(self):
525        self.load('body_io')
526
527        assert self.get()['body'] == '0123456789', 'body io'
528
529    def test_python_application_body_io_file(self):
530        self.load('body_io_file')
531
532        assert self.get()['body'] == 'body\n', 'body io file'
533
534    @pytest.mark.skip('not yet')
535    def test_python_application_syntax_error(self):
536        skip_alert(r'Python failed to import module "wsgi"')
537        self.load('syntax_error')
538
539        assert self.get()['status'] == 500, 'syntax error'
540
541    def test_python_application_loading_error(self):
542        skip_alert(r'Python failed to import module "blah"')
543
544        self.load('empty')
545
546        assert 'success' in self.conf('"blah"', 'applications/empty/module')
547
548        assert self.get()['status'] == 503, 'loading error'
549
550    def test_python_application_close(self):
551        self.load('close')
552
553        self.get()
554
555        unit_stop()
556
557        assert self.wait_for_record(r'Close called\.') is not None, 'close'
558
559    def test_python_application_close_error(self):
560        self.load('close_error')
561
562        self.get()
563
564        unit_stop()
565
566        assert (
567            self.wait_for_record(r'Close called\.') is not None
568        ), 'close error'
569
570    def test_python_application_not_iterable(self):
571        self.load('not_iterable')
572
573        self.get()
574
575        unit_stop()
576
577        assert (
578            self.wait_for_record(
579                r'\[error\].+the application returned not an iterable object'
580            )
581            is not None
582        ), 'not iterable'
583
584    def test_python_application_write(self):
585        self.load('write')
586
587        assert self.get()['body'] == '0123456789', 'write'
588
589    def test_python_application_threading(self):
590        """wait_for_record() timeouts after 5s while every thread works at
591        least 3s.  So without releasing GIL test should fail.
592        """
593
594        self.load('threading')
595
596        for _ in range(10):
597            self.get(no_recv=True)
598
599        assert (
600            self.wait_for_record(r'\(5\) Thread: 100') is not None
601        ), 'last thread finished'
602
603    def test_python_application_iter_exception(self):
604        self.load('iter_exception')
605
606        # Default request doesn't lead to the exception.
607
608        resp = self.get(
609            headers={
610                'Host': 'localhost',
611                'X-Skip': '9',
612                'X-Chunked': '1',
613                'Connection': 'close',
614            }
615        )
616        assert resp['status'] == 200, 'status'
617        assert resp['body'] == 'XXXXXXX', 'body'
618
619        # Exception before start_response().
620
621        assert self.get()['status'] == 503, 'error'
622
623        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
624        assert (
625            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
626            is not None
627        ), 'first exception raise'
628        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
629
630        # Exception after start_response(), before first write().
631
632        assert (
633            self.get(
634                headers={
635                    'Host': 'localhost',
636                    'X-Skip': '1',
637                    'Connection': 'close',
638                }
639            )['status']
640            == 503
641        ), 'error 2'
642
643        assert (
644            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
645            is not None
646        ), 'exception raise second'
647        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
648
649        # Exception after first write(), before first __next__().
650
651        _, sock = self.get(
652            headers={
653                'Host': 'localhost',
654                'X-Skip': '2',
655                'Connection': 'keep-alive',
656            },
657            start=True,
658        )
659
660        assert (
661            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
662            is not None
663        ), 'exception raise third'
664        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
665
666        assert self.get(sock=sock) == {}, 'closed connection'
667
668        # Exception after first write(), before first __next__(),
669        # chunked (incomplete body).
670
671        resp = self.get(
672            headers={
673                'Host': 'localhost',
674                'X-Skip': '2',
675                'X-Chunked': '1',
676                'Connection': 'close',
677            },
678            raw_resp=True,
679        )
680        if resp:
681            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
682        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
683
684        # Exception in __next__().
685
686        _, sock = self.get(
687            headers={
688                'Host': 'localhost',
689                'X-Skip': '3',
690                'Connection': 'keep-alive',
691            },
692            start=True,
693        )
694
695        assert (
696            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
697            is not None
698        ), 'exception raise next'
699        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
700
701        assert self.get(sock=sock) == {}, 'closed connection 2'
702
703        # Exception in __next__(), chunked (incomplete body).
704
705        resp = self.get(
706            headers={
707                'Host': 'localhost',
708                'X-Skip': '3',
709                'X-Chunked': '1',
710                'Connection': 'close',
711            },
712            raw_resp=True,
713        )
714        if resp:
715            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
716        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
717
718        # Exception before start_response() and in close().
719
720        assert (
721            self.get(
722                headers={
723                    'Host': 'localhost',
724                    'X-Not-Skip-Close': '1',
725                    'Connection': 'close',
726                }
727            )['status']
728            == 503
729        ), 'error'
730
731        assert (
732            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
733            is not None
734        ), 'exception raise close'
735        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
736
737    def test_python_user_group(self, is_su):
738        if not is_su:
739            pytest.skip('requires root')
740
741        nobody_uid = pwd.getpwnam('nobody').pw_uid
742
743        group = 'nobody'
744
745        try:
746            group_id = grp.getgrnam(group).gr_gid
747        except:
748            group = 'nogroup'
749            group_id = grp.getgrnam(group).gr_gid
750
751        self.load('user_group')
752
753        obj = self.getjson()['body']
754        assert obj['UID'] == nobody_uid, 'nobody uid'
755        assert obj['GID'] == group_id, 'nobody gid'
756
757        self.load('user_group', user='nobody')
758
759        obj = self.getjson()['body']
760        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
761        assert obj['GID'] == group_id, 'nobody gid user=nobody'
762
763        self.load('user_group', user='nobody', group=group)
764
765        obj = self.getjson()['body']
766        assert obj['UID'] == nobody_uid, (
767            'nobody uid user=nobody group=%s' % group
768        )
769
770        assert obj['GID'] == group_id, (
771            'nobody gid user=nobody group=%s' % group
772        )
773
774        self.load('user_group', group=group)
775
776        obj = self.getjson()['body']
777        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
778
779        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
780
781        self.load('user_group', user='root')
782
783        obj = self.getjson()['body']
784        assert obj['UID'] == 0, 'root uid user=root'
785        assert obj['GID'] == 0, 'root gid user=root'
786
787        group = 'root'
788
789        try:
790            grp.getgrnam(group)
791            group = True
792        except:
793            group = False
794
795        if group:
796            self.load('user_group', user='root', group='root')
797
798            obj = self.getjson()['body']
799            assert obj['UID'] == 0, 'root uid user=root group=root'
800            assert obj['GID'] == 0, 'root gid user=root group=root'
801
802            self.load('user_group', group='root')
803
804            obj = self.getjson()['body']
805            assert obj['UID'] == nobody_uid, 'root uid group=root'
806            assert obj['GID'] == 0, 'root gid group=root'
807
808    def test_python_application_callable(self):
809        skip_alert(r'Python failed to get "blah" from module')
810        self.load('callable')
811
812        assert self.get()['status'] == 204, 'default application response'
813
814        assert 'success' in self.conf(
815            '"app"', 'applications/callable/callable'
816        )
817
818        assert self.get()['status'] == 200, 'callable response'
819
820        assert 'success' in self.conf(
821            '"blah"', 'applications/callable/callable'
822        )
823
824        assert self.get()['status'] not in [200, 204], 'callable response inv'
825
826        assert 'success' in self.conf(
827            '"app"', 'applications/callable/callable'
828        )
829
830        assert self.get()['status'] == 200, 'callable response 2'
831
832        assert 'success' in self.conf_delete('applications/callable/callable')
833
834        assert self.get()['status'] == 204, 'default response 2'
835