xref: /unit/test/test_python_application.py (revision 1706:a1da56837554)
1import grp
2import pwd
3import re
4import time
5
6import pytest
7
8from conftest import option
9from conftest import skip_alert
10from conftest import unit_stop
11from unit.applications.lang.python import TestApplicationPython
12
13
14class TestPythonApplication(TestApplicationPython):
15    prerequisites = {'modules': {'python': 'all'}}
16
17    def findall(self, pattern):
18        with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f:
19            return re.findall(pattern, f.read())
20
21    def test_python_application_variables(self):
22        self.load('variables')
23
24        body = 'Test body string.'
25
26        resp = self.http(
27            b"""POST / HTTP/1.1
28Host: localhost
29Content-Length: %d
30Custom-Header: blah
31Custom-hEader: Blah
32Content-Type: text/html
33Connection: close
34custom-header: BLAH
35
36%s""" % (len(body), body.encode()),
37            raw=True,
38        )
39
40        assert resp['status'] == 200, 'status'
41        headers = resp['headers']
42        header_server = headers.pop('Server')
43        assert re.search(r'Unit/[\d\.]+', header_server), 'server header'
44        assert (
45            headers.pop('Server-Software') == header_server
46        ), 'server software header'
47
48        date = headers.pop('Date')
49        assert date[-4:] == ' GMT', 'date header timezone'
50        assert (
51            abs(self.date_to_sec_epoch(date) - self.sec_epoch()) < 5
52        ), 'date header'
53
54        assert headers == {
55            'Connection': 'close',
56            'Content-Length': str(len(body)),
57            'Content-Type': 'text/html',
58            'Request-Method': 'POST',
59            'Request-Uri': '/',
60            'Http-Host': 'localhost',
61            'Server-Protocol': 'HTTP/1.1',
62            'Custom-Header': 'blah, Blah, BLAH',
63            'Wsgi-Version': '(1, 0)',
64            'Wsgi-Url-Scheme': 'http',
65            'Wsgi-Multithread': 'False',
66            'Wsgi-Multiprocess': 'True',
67            'Wsgi-Run-Once': 'False',
68        }, 'headers'
69        assert resp['body'] == body, 'body'
70
71    def test_python_application_query_string(self):
72        self.load('query_string')
73
74        resp = self.get(url='/?var1=val1&var2=val2')
75
76        assert (
77            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
78        ), 'Query-String header'
79
80    def test_python_application_query_string_space(self):
81        self.load('query_string')
82
83        resp = self.get(url='/ ?var1=val1&var2=val2')
84        assert (
85            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
86        ), 'Query-String space'
87
88        resp = self.get(url='/ %20?var1=val1&var2=val2')
89        assert (
90            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
91        ), 'Query-String space 2'
92
93        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
94        assert (
95            resp['headers']['Query-String'] == 'var1=val1&var2=val2'
96        ), 'Query-String space 3'
97
98        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
99        assert (
100            resp['headers']['Query-String'] == ' var1= val1 & var2=val2'
101        ), 'Query-String space 4'
102
103    def test_python_application_query_string_empty(self):
104        self.load('query_string')
105
106        resp = self.get(url='/?')
107
108        assert resp['status'] == 200, 'query string empty status'
109        assert resp['headers']['Query-String'] == '', 'query string empty'
110
111    def test_python_application_query_string_absent(self):
112        self.load('query_string')
113
114        resp = self.get()
115
116        assert resp['status'] == 200, 'query string absent status'
117        assert resp['headers']['Query-String'] == '', 'query string absent'
118
119    @pytest.mark.skip('not yet')
120    def test_python_application_server_port(self):
121        self.load('server_port')
122
123        assert (
124            self.get()['headers']['Server-Port'] == '7080'
125        ), 'Server-Port header'
126
127    @pytest.mark.skip('not yet')
128    def test_python_application_working_directory_invalid(self):
129        self.load('empty')
130
131        assert 'success' in self.conf(
132            '"/blah"', 'applications/empty/working_directory'
133        ), 'configure invalid working_directory'
134
135        assert self.get()['status'] == 500, 'status'
136
137    def test_python_application_204_transfer_encoding(self):
138        self.load('204_no_content')
139
140        assert (
141            'Transfer-Encoding' not in self.get()['headers']
142        ), '204 header transfer encoding'
143
144    def test_python_application_ctx_iter_atexit(self):
145        self.load('ctx_iter_atexit')
146
147        resp = self.post(
148            headers={
149                'Host': 'localhost',
150                'Connection': 'close',
151                'Content-Type': 'text/html',
152            },
153            body='0123456789',
154        )
155
156        assert resp['status'] == 200, 'ctx iter status'
157        assert resp['body'] == '0123456789', 'ctx iter body'
158
159        self.conf({"listeners": {}, "applications": {}})
160
161        unit_stop()
162
163        assert (
164            self.wait_for_record(r'RuntimeError') is not None
165        ), 'ctx iter atexit'
166
167    def test_python_keepalive_body(self):
168        self.load('mirror')
169
170        assert self.get()['status'] == 200, 'init'
171
172        body = '0123456789' * 500
173        (resp, sock) = self.post(
174            headers={
175                'Host': 'localhost',
176                'Connection': 'keep-alive',
177                'Content-Type': 'text/html',
178            },
179            start=True,
180            body=body,
181            read_timeout=1,
182        )
183
184        assert resp['body'] == body, 'keep-alive 1'
185
186        body = '0123456789'
187        resp = self.post(
188            headers={
189                'Host': 'localhost',
190                'Connection': 'close',
191                'Content-Type': 'text/html',
192            },
193            sock=sock,
194            body=body,
195        )
196
197        assert resp['body'] == body, 'keep-alive 2'
198
199    def test_python_keepalive_reconfigure(self):
200        self.load('mirror')
201
202        assert self.get()['status'] == 200, 'init'
203
204        body = '0123456789'
205        conns = 3
206        socks = []
207
208        for i in range(conns):
209            (resp, sock) = self.post(
210                headers={
211                    'Host': 'localhost',
212                    'Connection': 'keep-alive',
213                    'Content-Type': 'text/html',
214                },
215                start=True,
216                body=body,
217                read_timeout=1,
218            )
219
220            assert resp['body'] == body, 'keep-alive open'
221
222            self.load('mirror', processes=i + 1)
223
224            socks.append(sock)
225
226        for i in range(conns):
227            (resp, sock) = self.post(
228                headers={
229                    'Host': 'localhost',
230                    'Connection': 'keep-alive',
231                    'Content-Type': 'text/html',
232                },
233                start=True,
234                sock=socks[i],
235                body=body,
236                read_timeout=1,
237            )
238
239            assert resp['body'] == body, 'keep-alive request'
240
241            self.load('mirror', processes=i + 1)
242
243        for i in range(conns):
244            resp = self.post(
245                headers={
246                    'Host': 'localhost',
247                    'Connection': 'close',
248                    'Content-Type': 'text/html',
249                },
250                sock=socks[i],
251                body=body,
252            )
253
254            assert resp['body'] == body, 'keep-alive close'
255
256            self.load('mirror', processes=i + 1)
257
258    def test_python_keepalive_reconfigure_2(self):
259        self.load('mirror')
260
261        assert self.get()['status'] == 200, 'init'
262
263        body = '0123456789'
264
265        (resp, sock) = self.post(
266            headers={
267                'Host': 'localhost',
268                'Connection': 'keep-alive',
269                'Content-Type': 'text/html',
270            },
271            start=True,
272            body=body,
273            read_timeout=1,
274        )
275
276        assert resp['body'] == body, 'reconfigure 2 keep-alive 1'
277
278        self.load('empty')
279
280        assert self.get()['status'] == 200, 'init'
281
282        (resp, sock) = self.post(
283            headers={
284                'Host': 'localhost',
285                'Connection': 'close',
286                'Content-Type': 'text/html',
287            },
288            start=True,
289            sock=sock,
290            body=body,
291        )
292
293        assert resp['status'] == 200, 'reconfigure 2 keep-alive 2'
294        assert resp['body'] == '', 'reconfigure 2 keep-alive 2 body'
295
296        assert 'success' in self.conf(
297            {"listeners": {}, "applications": {}}
298        ), 'reconfigure 2 clear configuration'
299
300        resp = self.get(sock=sock)
301
302        assert resp == {}, 'reconfigure 2 keep-alive 3'
303
304    def test_python_keepalive_reconfigure_3(self):
305        self.load('empty')
306
307        assert self.get()['status'] == 200, 'init'
308
309        (_, sock) = self.http(
310            b"""GET / HTTP/1.1
311""",
312            start=True,
313            raw=True,
314            no_recv=True,
315        )
316
317        assert self.get()['status'] == 200
318
319        assert 'success' in self.conf(
320            {"listeners": {}, "applications": {}}
321        ), 'reconfigure 3 clear configuration'
322
323        resp = self.http(
324            b"""Host: localhost
325Connection: close
326
327""",
328            sock=sock,
329            raw=True,
330        )
331
332        assert resp['status'] == 200, 'reconfigure 3'
333
334    def test_python_atexit(self):
335        self.load('atexit')
336
337        self.get()
338
339        self.conf({"listeners": {}, "applications": {}})
340
341        unit_stop()
342
343        assert self.wait_for_record(r'At exit called\.') is not None, 'atexit'
344
345    def test_python_process_switch(self):
346        self.load('delayed', processes=2)
347
348        self.get(
349            headers={
350                'Host': 'localhost',
351                'Content-Length': '0',
352                'X-Delay': '5',
353                'Connection': 'close',
354            },
355            no_recv=True,
356        )
357
358        headers_delay_1 = {
359            'Connection': 'close',
360            'Host': 'localhost',
361            'Content-Length': '0',
362            'X-Delay': '1',
363        }
364
365        self.get(headers=headers_delay_1, no_recv=True)
366
367        time.sleep(0.5)
368
369        for _ in range(10):
370            self.get(headers=headers_delay_1, no_recv=True)
371
372        self.get(headers=headers_delay_1)
373
374    @pytest.mark.skip('not yet')
375    def test_python_application_start_response_exit(self):
376        self.load('start_response_exit')
377
378        assert self.get()['status'] == 500, 'start response exit'
379
380    def test_python_application_input_iter(self):
381        self.load('input_iter')
382
383        body = '''0123456789
384next line
385
386last line'''
387
388        resp = self.post(body=body)
389        assert resp['body'] == body, 'input iter'
390        assert resp['headers']['X-Lines-Count'] == '4', 'input iter lines'
391
392    def test_python_application_input_readline(self):
393        self.load('input_readline')
394
395        body = '''0123456789
396next line
397
398last line'''
399
400        resp = self.post(body=body)
401        assert resp['body'] == body, 'input readline'
402        assert resp['headers']['X-Lines-Count'] == '4', 'input readline lines'
403
404    def test_python_application_input_readline_size(self):
405        self.load('input_readline_size')
406
407        body = '''0123456789
408next line
409
410last line'''
411
412        assert self.post(body=body)['body'] == body, 'input readline size'
413        assert (
414            self.post(body='0123')['body'] == '0123'
415        ), 'input readline size less'
416
417    def test_python_application_input_readlines(self):
418        self.load('input_readlines')
419
420        body = '''0123456789
421next line
422
423last line'''
424
425        resp = self.post(body=body)
426        assert resp['body'] == body, 'input readlines'
427        assert resp['headers']['X-Lines-Count'] == '4', 'input readlines lines'
428
429    def test_python_application_input_readlines_huge(self):
430        self.load('input_readlines')
431
432        body = (
433            '''0123456789 abcdefghi
434next line: 0123456789 abcdefghi
435
436last line: 987654321
437'''
438            * 512
439        )
440
441        assert (
442            self.post(body=body, read_buffer_size=16384)['body'] == body
443        ), 'input readlines huge'
444
445    def test_python_application_input_read_length(self):
446        self.load('input_read_length')
447
448        body = '0123456789'
449
450        resp = self.post(
451            headers={
452                'Host': 'localhost',
453                'Input-Length': '5',
454                'Connection': 'close',
455            },
456            body=body,
457        )
458
459        assert resp['body'] == body[:5], 'input read length lt body'
460
461        resp = self.post(
462            headers={
463                'Host': 'localhost',
464                'Input-Length': '15',
465                'Connection': 'close',
466            },
467            body=body,
468        )
469
470        assert resp['body'] == body, 'input read length gt body'
471
472        resp = self.post(
473            headers={
474                'Host': 'localhost',
475                'Input-Length': '0',
476                'Connection': 'close',
477            },
478            body=body,
479        )
480
481        assert resp['body'] == '', 'input read length zero'
482
483        resp = self.post(
484            headers={
485                'Host': 'localhost',
486                'Input-Length': '-1',
487                'Connection': 'close',
488            },
489            body=body,
490        )
491
492        assert resp['body'] == body, 'input read length negative'
493
494    @pytest.mark.skip('not yet')
495    def test_python_application_errors_write(self):
496        self.load('errors_write')
497
498        self.get()
499
500        unit_stop()
501
502        assert (
503            self.wait_for_record(r'\[error\].+Error in application\.')
504            is not None
505        ), 'errors write'
506
507    def test_python_application_body_array(self):
508        self.load('body_array')
509
510        assert self.get()['body'] == '0123456789', 'body array'
511
512    def test_python_application_body_io(self):
513        self.load('body_io')
514
515        assert self.get()['body'] == '0123456789', 'body io'
516
517    def test_python_application_body_io_file(self):
518        self.load('body_io_file')
519
520        assert self.get()['body'] == 'body\n', 'body io file'
521
522    @pytest.mark.skip('not yet')
523    def test_python_application_syntax_error(self):
524        skip_alert(r'Python failed to import module "wsgi"')
525        self.load('syntax_error')
526
527        assert self.get()['status'] == 500, 'syntax error'
528
529    def test_python_application_loading_error(self):
530        skip_alert(r'Python failed to import module "blah"')
531
532        self.load('empty', module="blah")
533
534        assert self.get()['status'] == 503, 'loading error'
535
536    def test_python_application_close(self):
537        self.load('close')
538
539        self.get()
540
541        unit_stop()
542
543        assert self.wait_for_record(r'Close called\.') is not None, 'close'
544
545    def test_python_application_close_error(self):
546        self.load('close_error')
547
548        self.get()
549
550        unit_stop()
551
552        assert (
553            self.wait_for_record(r'Close called\.') is not None
554        ), 'close error'
555
556    def test_python_application_not_iterable(self):
557        self.load('not_iterable')
558
559        self.get()
560
561        unit_stop()
562
563        assert (
564            self.wait_for_record(
565                r'\[error\].+the application returned not an iterable object'
566            )
567            is not None
568        ), 'not iterable'
569
570    def test_python_application_write(self):
571        self.load('write')
572
573        assert self.get()['body'] == '0123456789', 'write'
574
575    def test_python_application_threading(self):
576        """wait_for_record() timeouts after 5s while every thread works at
577        least 3s.  So without releasing GIL test should fail.
578        """
579
580        self.load('threading')
581
582        for _ in range(10):
583            self.get(no_recv=True)
584
585        assert (
586            self.wait_for_record(r'\(5\) Thread: 100') is not None
587        ), 'last thread finished'
588
589    def test_python_application_iter_exception(self):
590        self.load('iter_exception')
591
592        # Default request doesn't lead to the exception.
593
594        resp = self.get(
595            headers={
596                'Host': 'localhost',
597                'X-Skip': '9',
598                'X-Chunked': '1',
599                'Connection': 'close',
600            }
601        )
602        assert resp['status'] == 200, 'status'
603        assert resp['body'] == 'XXXXXXX', 'body'
604
605        # Exception before start_response().
606
607        assert self.get()['status'] == 503, 'error'
608
609        assert self.wait_for_record(r'Traceback') is not None, 'traceback'
610        assert (
611            self.wait_for_record(r'raise Exception\(\'first exception\'\)')
612            is not None
613        ), 'first exception raise'
614        assert len(self.findall(r'Traceback')) == 1, 'traceback count 1'
615
616        # Exception after start_response(), before first write().
617
618        assert (
619            self.get(
620                headers={
621                    'Host': 'localhost',
622                    'X-Skip': '1',
623                    'Connection': 'close',
624                }
625            )['status']
626            == 503
627        ), 'error 2'
628
629        assert (
630            self.wait_for_record(r'raise Exception\(\'second exception\'\)')
631            is not None
632        ), 'exception raise second'
633        assert len(self.findall(r'Traceback')) == 2, 'traceback count 2'
634
635        # Exception after first write(), before first __next__().
636
637        _, sock = self.get(
638            headers={
639                'Host': 'localhost',
640                'X-Skip': '2',
641                'Connection': 'keep-alive',
642            },
643            start=True,
644        )
645
646        assert (
647            self.wait_for_record(r'raise Exception\(\'third exception\'\)')
648            is not None
649        ), 'exception raise third'
650        assert len(self.findall(r'Traceback')) == 3, 'traceback count 3'
651
652        assert self.get(sock=sock) == {}, 'closed connection'
653
654        # Exception after first write(), before first __next__(),
655        # chunked (incomplete body).
656
657        resp = self.get(
658            headers={
659                'Host': 'localhost',
660                'X-Skip': '2',
661                'X-Chunked': '1',
662                'Connection': 'close',
663            },
664            raw_resp=True,
665        )
666        if resp:
667            assert resp[-5:] != '0\r\n\r\n', 'incomplete body'
668        assert len(self.findall(r'Traceback')) == 4, 'traceback count 4'
669
670        # Exception in __next__().
671
672        _, sock = self.get(
673            headers={
674                'Host': 'localhost',
675                'X-Skip': '3',
676                'Connection': 'keep-alive',
677            },
678            start=True,
679        )
680
681        assert (
682            self.wait_for_record(r'raise Exception\(\'next exception\'\)')
683            is not None
684        ), 'exception raise next'
685        assert len(self.findall(r'Traceback')) == 5, 'traceback count 5'
686
687        assert self.get(sock=sock) == {}, 'closed connection 2'
688
689        # Exception in __next__(), chunked (incomplete body).
690
691        resp = self.get(
692            headers={
693                'Host': 'localhost',
694                'X-Skip': '3',
695                'X-Chunked': '1',
696                'Connection': 'close',
697            },
698            raw_resp=True,
699        )
700        if resp:
701            assert resp[-5:] != '0\r\n\r\n', 'incomplete body 2'
702        assert len(self.findall(r'Traceback')) == 6, 'traceback count 6'
703
704        # Exception before start_response() and in close().
705
706        assert (
707            self.get(
708                headers={
709                    'Host': 'localhost',
710                    'X-Not-Skip-Close': '1',
711                    'Connection': 'close',
712                }
713            )['status']
714            == 503
715        ), 'error'
716
717        assert (
718            self.wait_for_record(r'raise Exception\(\'close exception\'\)')
719            is not None
720        ), 'exception raise close'
721        assert len(self.findall(r'Traceback')) == 8, 'traceback count 8'
722
723    def test_python_user_group(self, is_su):
724        if not is_su:
725            pytest.skip('requires root')
726
727        nobody_uid = pwd.getpwnam('nobody').pw_uid
728
729        group = 'nobody'
730
731        try:
732            group_id = grp.getgrnam(group).gr_gid
733        except KeyError:
734            group = 'nogroup'
735            group_id = grp.getgrnam(group).gr_gid
736
737        self.load('user_group')
738
739        obj = self.getjson()['body']
740        assert obj['UID'] == nobody_uid, 'nobody uid'
741        assert obj['GID'] == group_id, 'nobody gid'
742
743        self.load('user_group', user='nobody')
744
745        obj = self.getjson()['body']
746        assert obj['UID'] == nobody_uid, 'nobody uid user=nobody'
747        assert obj['GID'] == group_id, 'nobody gid user=nobody'
748
749        self.load('user_group', user='nobody', group=group)
750
751        obj = self.getjson()['body']
752        assert obj['UID'] == nobody_uid, (
753            'nobody uid user=nobody group=%s' % group
754        )
755
756        assert obj['GID'] == group_id, (
757            'nobody gid user=nobody group=%s' % group
758        )
759
760        self.load('user_group', group=group)
761
762        obj = self.getjson()['body']
763        assert obj['UID'] == nobody_uid, 'nobody uid group=%s' % group
764
765        assert obj['GID'] == group_id, 'nobody gid group=%s' % group
766
767        self.load('user_group', user='root')
768
769        obj = self.getjson()['body']
770        assert obj['UID'] == 0, 'root uid user=root'
771        assert obj['GID'] == 0, 'root gid user=root'
772
773        group = 'root'
774
775        try:
776            grp.getgrnam(group)
777            group = True
778        except KeyError:
779            group = False
780
781        if group:
782            self.load('user_group', user='root', group='root')
783
784            obj = self.getjson()['body']
785            assert obj['UID'] == 0, 'root uid user=root group=root'
786            assert obj['GID'] == 0, 'root gid user=root group=root'
787
788            self.load('user_group', group='root')
789
790            obj = self.getjson()['body']
791            assert obj['UID'] == nobody_uid, 'root uid group=root'
792            assert obj['GID'] == 0, 'root gid group=root'
793
794    def test_python_application_callable(self):
795        skip_alert(r'Python failed to get "blah" from module')
796        self.load('callable')
797
798        assert self.get()['status'] == 204, 'default application response'
799
800        self.load('callable', callable="app")
801
802        assert self.get()['status'] == 200, 'callable response'
803
804        self.load('callable', callable="blah")
805
806        assert self.get()['status'] not in [200, 204], 'callable response inv'
807
808    def test_python_application_threads(self):
809        self.load('threads', threads=4)
810
811        socks = []
812
813        for i in range(4):
814            (_, sock) = self.get(
815                headers={
816                    'Host': 'localhost',
817                    'X-Delay': '2',
818                    'Connection': 'close',
819                },
820                no_recv=True,
821                start=True,
822            )
823
824            socks.append(sock)
825
826        threads = set()
827
828        for sock in socks:
829            resp = self.recvall(sock).decode('utf-8')
830
831            self.log_in(resp)
832
833            resp = self._resp_to_dict(resp)
834
835            assert resp['status'] == 200, 'status'
836
837            threads.add(resp['headers']['X-Thread'])
838
839            assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread'
840
841            sock.close()
842
843        assert len(socks) == len(threads), 'threads differs'
844