xref: /unit/test/test_python_application.py (revision 1467:195fe0a92670)
1import re
2import os
3import grp
4import pwd
5import time
6import unittest
7from unit.applications.lang.python import TestApplicationPython
8
9
10class TestPythonApplication(TestApplicationPython):
11    prerequisites = {'modules': {'python': 'all'}}
12
13    def findall(self, pattern):
14        with open(self.testdir + '/unit.log', 'r', errors='ignore') as f:
15            return re.findall(pattern, f.read())
16
17    def test_python_application_variables(self):
18        self.load('variables')
19
20        body = 'Test body string.'
21
22        resp = self.post(
23            headers={
24                'Host': 'localhost',
25                'Content-Type': 'text/html',
26                'Custom-Header': 'blah',
27                'Connection': 'close',
28            },
29            body=body,
30        )
31
32        self.assertEqual(resp['status'], 200, 'status')
33        headers = resp['headers']
34        header_server = headers.pop('Server')
35        self.assertRegex(header_server, r'Unit/[\d\.]+', 'server header')
36        self.assertEqual(
37            headers.pop('Server-Software'),
38            header_server,
39            'server software header',
40        )
41
42        date = headers.pop('Date')
43        self.assertEqual(date[-4:], ' GMT', 'date header timezone')
44        self.assertLess(
45            abs(self.date_to_sec_epoch(date) - self.sec_epoch()),
46            5,
47            'date header',
48        )
49
50        self.assertDictEqual(
51            headers,
52            {
53                'Connection': 'close',
54                'Content-Length': str(len(body)),
55                'Content-Type': 'text/html',
56                'Request-Method': 'POST',
57                'Request-Uri': '/',
58                'Http-Host': 'localhost',
59                'Server-Protocol': 'HTTP/1.1',
60                'Custom-Header': 'blah',
61                'Wsgi-Version': '(1, 0)',
62                'Wsgi-Url-Scheme': 'http',
63                'Wsgi-Multithread': 'False',
64                'Wsgi-Multiprocess': 'True',
65                'Wsgi-Run-Once': 'False',
66            },
67            'headers',
68        )
69        self.assertEqual(resp['body'], body, 'body')
70
71    def test_python_application_query_string(self):
72        self.load('query_string')
73
74        resp = self.get(url='/?var1=val1&var2=val2')
75
76        self.assertEqual(
77            resp['headers']['Query-String'],
78            'var1=val1&var2=val2',
79            'Query-String header',
80        )
81
82    def test_python_application_query_string_space(self):
83        self.load('query_string')
84
85        resp = self.get(url='/ ?var1=val1&var2=val2')
86        self.assertEqual(
87            resp['headers']['Query-String'],
88            'var1=val1&var2=val2',
89            'Query-String space',
90        )
91
92        resp = self.get(url='/ %20?var1=val1&var2=val2')
93        self.assertEqual(
94            resp['headers']['Query-String'],
95            'var1=val1&var2=val2',
96            'Query-String space 2',
97        )
98
99        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
100        self.assertEqual(
101            resp['headers']['Query-String'],
102            'var1=val1&var2=val2',
103            'Query-String space 3',
104        )
105
106        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
107        self.assertEqual(
108            resp['headers']['Query-String'],
109            ' var1= val1 & var2=val2',
110            'Query-String space 4',
111        )
112
113    def test_python_application_query_string_empty(self):
114        self.load('query_string')
115
116        resp = self.get(url='/?')
117
118        self.assertEqual(resp['status'], 200, 'query string empty status')
119        self.assertEqual(
120            resp['headers']['Query-String'], '', 'query string empty'
121        )
122
123    def test_python_application_query_string_absent(self):
124        self.load('query_string')
125
126        resp = self.get()
127
128        self.assertEqual(resp['status'], 200, 'query string absent status')
129        self.assertEqual(
130            resp['headers']['Query-String'], '', 'query string absent'
131        )
132
133    @unittest.skip('not yet')
134    def test_python_application_server_port(self):
135        self.load('server_port')
136
137        self.assertEqual(
138            self.get()['headers']['Server-Port'], '7080', 'Server-Port header'
139        )
140
141    @unittest.skip('not yet')
142    def test_python_application_working_directory_invalid(self):
143        self.load('empty')
144
145        self.assertIn(
146            'success',
147            self.conf('"/blah"', 'applications/empty/working_directory'),
148            'configure invalid working_directory',
149        )
150
151        self.assertEqual(self.get()['status'], 500, 'status')
152
153    def test_python_application_204_transfer_encoding(self):
154        self.load('204_no_content')
155
156        self.assertNotIn(
157            'Transfer-Encoding',
158            self.get()['headers'],
159            '204 header transfer encoding',
160        )
161
162    def test_python_application_ctx_iter_atexit(self):
163        self.load('ctx_iter_atexit')
164
165        resp = self.post(
166            headers={
167                'Host': 'localhost',
168                'Connection': 'close',
169                'Content-Type': 'text/html',
170            },
171            body='0123456789',
172        )
173
174        self.assertEqual(resp['status'], 200, 'ctx iter status')
175        self.assertEqual(resp['body'], '0123456789', 'ctx iter body')
176
177        self.conf({"listeners": {}, "applications": {}})
178
179        self.stop()
180
181        self.assertIsNotNone(
182            self.wait_for_record(r'RuntimeError'), 'ctx iter atexit'
183        )
184
185    def test_python_keepalive_body(self):
186        self.load('mirror')
187
188        self.assertEqual(self.get()['status'], 200, 'init')
189
190        body = '0123456789' * 500
191        (resp, sock) = self.post(
192            headers={
193                'Host': 'localhost',
194                'Connection': 'keep-alive',
195                'Content-Type': 'text/html',
196            },
197            start=True,
198            body=body,
199            read_timeout=1,
200        )
201
202        self.assertEqual(resp['body'], body, 'keep-alive 1')
203
204        body = '0123456789'
205        resp = self.post(
206            headers={
207                'Host': 'localhost',
208                'Connection': 'close',
209                'Content-Type': 'text/html',
210            },
211            sock=sock,
212            body=body,
213        )
214
215        self.assertEqual(resp['body'], body, 'keep-alive 2')
216
217    def test_python_keepalive_reconfigure(self):
218        self.skip_alerts.extend(
219            [
220                r'pthread_mutex.+failed',
221                r'failed to apply',
222                r'process \d+ exited on signal',
223            ]
224        )
225        self.load('mirror')
226
227        self.assertEqual(self.get()['status'], 200, 'init')
228
229        body = '0123456789'
230        conns = 3
231        socks = []
232
233        for i in range(conns):
234            (resp, sock) = self.post(
235                headers={
236                    'Host': 'localhost',
237                    'Connection': 'keep-alive',
238                    'Content-Type': 'text/html',
239                },
240                start=True,
241                body=body,
242                read_timeout=1,
243            )
244
245            self.assertEqual(resp['body'], body, 'keep-alive open')
246            self.assertIn(
247                'success',
248                self.conf(str(i + 1), 'applications/mirror/processes'),
249                'reconfigure',
250            )
251
252            socks.append(sock)
253
254        for i in range(conns):
255            (resp, sock) = self.post(
256                headers={
257                    'Host': 'localhost',
258                    'Connection': 'keep-alive',
259                    'Content-Type': 'text/html',
260                },
261                start=True,
262                sock=socks[i],
263                body=body,
264                read_timeout=1,
265            )
266
267            self.assertEqual(resp['body'], body, 'keep-alive request')
268            self.assertIn(
269                'success',
270                self.conf(str(i + 1), 'applications/mirror/processes'),
271                'reconfigure 2',
272            )
273
274        for i in range(conns):
275            resp = self.post(
276                headers={
277                    'Host': 'localhost',
278                    'Connection': 'close',
279                    'Content-Type': 'text/html',
280                },
281                sock=socks[i],
282                body=body,
283            )
284
285            self.assertEqual(resp['body'], body, 'keep-alive close')
286            self.assertIn(
287                'success',
288                self.conf(str(i + 1), 'applications/mirror/processes'),
289                'reconfigure 3',
290            )
291
292    def test_python_keepalive_reconfigure_2(self):
293        self.load('mirror')
294
295        self.assertEqual(self.get()['status'], 200, 'init')
296
297        body = '0123456789'
298
299        (resp, sock) = self.post(
300            headers={
301                'Host': 'localhost',
302                'Connection': 'keep-alive',
303                'Content-Type': 'text/html',
304            },
305            start=True,
306            body=body,
307            read_timeout=1,
308        )
309
310        self.assertEqual(resp['body'], body, 'reconfigure 2 keep-alive 1')
311
312        self.load('empty')
313
314        self.assertEqual(self.get()['status'], 200, 'init')
315
316        (resp, sock) = self.post(
317            headers={
318                'Host': 'localhost',
319                'Connection': 'close',
320                'Content-Type': 'text/html',
321            },
322            start=True,
323            sock=sock,
324            body=body,
325        )
326
327        self.assertEqual(resp['status'], 200, 'reconfigure 2 keep-alive 2')
328        self.assertEqual(resp['body'], '', 'reconfigure 2 keep-alive 2 body')
329
330        self.assertIn(
331            'success',
332            self.conf({"listeners": {}, "applications": {}}),
333            'reconfigure 2 clear configuration',
334        )
335
336        resp = self.get(sock=sock)
337
338        self.assertEqual(resp, {}, 'reconfigure 2 keep-alive 3')
339
340    def test_python_keepalive_reconfigure_3(self):
341        self.load('empty')
342
343        self.assertEqual(self.get()['status'], 200, 'init')
344
345        (_, sock) = self.http(
346            b"""GET / HTTP/1.1
347""",
348            start=True,
349            raw=True,
350            no_recv=True,
351        )
352
353        self.assertEqual(self.get()['status'], 200)
354
355        self.assertIn(
356            'success',
357            self.conf({"listeners": {}, "applications": {}}),
358            'reconfigure 3 clear configuration',
359        )
360
361        resp = self.http(
362            b"""Host: localhost
363Connection: close
364
365""",
366            sock=sock,
367            raw=True,
368        )
369
370        self.assertEqual(resp['status'], 200, 'reconfigure 3')
371
372    def test_python_atexit(self):
373        self.load('atexit')
374
375        self.get()
376
377        self.conf({"listeners": {}, "applications": {}})
378
379        self.stop()
380
381        self.assertIsNotNone(
382            self.wait_for_record(r'At exit called\.'), 'atexit'
383        )
384
385    def test_python_process_switch(self):
386        self.load('delayed')
387
388        self.assertIn(
389            'success',
390            self.conf('2', 'applications/delayed/processes'),
391            'configure 2 processes',
392        )
393
394        self.get(headers={
395            'Host': 'localhost',
396            'Content-Length': '0',
397            'X-Delay': '5',
398            'Connection': 'close',
399        }, no_recv=True)
400
401        headers_delay_1 = {
402            'Connection': 'close',
403            'Host': 'localhost',
404            'Content-Length': '0',
405            'X-Delay': '1',
406        }
407
408        self.get(headers=headers_delay_1, no_recv=True)
409
410        time.sleep(0.5)
411
412        for _ in range(10):
413            self.get(headers=headers_delay_1, no_recv=True)
414
415        self.get(headers=headers_delay_1)
416
417    @unittest.skip('not yet')
418    def test_python_application_start_response_exit(self):
419        self.load('start_response_exit')
420
421        self.assertEqual(self.get()['status'], 500, 'start response exit')
422
423    def test_python_application_input_iter(self):
424        self.load('input_iter')
425
426        body = '''0123456789
427next line
428
429last line'''
430
431        resp = self.post(body=body)
432        self.assertEqual(resp['body'], body, 'input iter')
433        self.assertEqual(
434            resp['headers']['X-Lines-Count'], '4', 'input iter lines'
435        )
436
437    def test_python_application_input_readline(self):
438        self.load('input_readline')
439
440        body = '''0123456789
441next line
442
443last line'''
444
445        resp = self.post(body=body)
446        self.assertEqual(resp['body'], body, 'input readline')
447        self.assertEqual(
448            resp['headers']['X-Lines-Count'], '4', 'input readline lines'
449        )
450
451    def test_python_application_input_readline_size(self):
452        self.load('input_readline_size')
453
454        body = '''0123456789
455next line
456
457last line'''
458
459        self.assertEqual(
460            self.post(body=body)['body'], body, 'input readline size'
461        )
462        self.assertEqual(
463            self.post(body='0123')['body'], '0123', 'input readline size less'
464        )
465
466    def test_python_application_input_readlines(self):
467        self.load('input_readlines')
468
469        body = '''0123456789
470next line
471
472last line'''
473
474        resp = self.post(body=body)
475        self.assertEqual(resp['body'], body, 'input readlines')
476        self.assertEqual(
477            resp['headers']['X-Lines-Count'], '4', 'input readlines lines'
478        )
479
480    def test_python_application_input_readlines_huge(self):
481        self.load('input_readlines')
482
483        body = (
484            '''0123456789 abcdefghi
485next line: 0123456789 abcdefghi
486
487last line: 987654321
488'''
489            * 512
490        )
491
492        self.assertEqual(
493            self.post(body=body, read_buffer_size=16384)['body'],
494            body,
495            'input readlines huge',
496        )
497
498    def test_python_application_input_read_length(self):
499        self.load('input_read_length')
500
501        body = '0123456789'
502
503        resp = self.post(
504            headers={
505                'Host': 'localhost',
506                'Input-Length': '5',
507                'Connection': 'close',
508            },
509            body=body,
510        )
511
512        self.assertEqual(resp['body'], body[:5], 'input read length lt body')
513
514        resp = self.post(
515            headers={
516                'Host': 'localhost',
517                'Input-Length': '15',
518                'Connection': 'close',
519            },
520            body=body,
521        )
522
523        self.assertEqual(resp['body'], body, 'input read length gt body')
524
525        resp = self.post(
526            headers={
527                'Host': 'localhost',
528                'Input-Length': '0',
529                'Connection': 'close',
530            },
531            body=body,
532        )
533
534        self.assertEqual(resp['body'], '', 'input read length zero')
535
536        resp = self.post(
537            headers={
538                'Host': 'localhost',
539                'Input-Length': '-1',
540                'Connection': 'close',
541            },
542            body=body,
543        )
544
545        self.assertEqual(resp['body'], body, 'input read length negative')
546
547    @unittest.skip('not yet')
548    def test_python_application_errors_write(self):
549        self.load('errors_write')
550
551        self.get()
552
553        self.stop()
554
555        self.assertIsNotNone(
556            self.wait_for_record(r'\[error\].+Error in application\.'),
557            'errors write',
558        )
559
560    def test_python_application_body_array(self):
561        self.load('body_array')
562
563        self.assertEqual(self.get()['body'], '0123456789', 'body array')
564
565    def test_python_application_body_io(self):
566        self.load('body_io')
567
568        self.assertEqual(self.get()['body'], '0123456789', 'body io')
569
570    def test_python_application_body_io_file(self):
571        self.load('body_io_file')
572
573        self.assertEqual(self.get()['body'], 'body\n', 'body io file')
574
575    @unittest.skip('not yet')
576    def test_python_application_syntax_error(self):
577        self.skip_alerts.append(r'Python failed to import module "wsgi"')
578        self.load('syntax_error')
579
580        self.assertEqual(self.get()['status'], 500, 'syntax error')
581
582    def test_python_application_close(self):
583        self.load('close')
584
585        self.get()
586
587        self.stop()
588
589        self.assertIsNotNone(self.wait_for_record(r'Close called\.'), 'close')
590
591    def test_python_application_close_error(self):
592        self.load('close_error')
593
594        self.get()
595
596        self.stop()
597
598        self.assertIsNotNone(
599            self.wait_for_record(r'Close called\.'), 'close error'
600        )
601
602    def test_python_application_not_iterable(self):
603        self.load('not_iterable')
604
605        self.get()
606
607        self.stop()
608
609        self.assertIsNotNone(
610            self.wait_for_record(
611                r'\[error\].+the application returned not an iterable object'
612            ),
613            'not iterable',
614        )
615
616    def test_python_application_write(self):
617        self.load('write')
618
619        self.assertEqual(self.get()['body'], '0123456789', 'write')
620
621    def test_python_application_threading(self):
622        """wait_for_record() timeouts after 5s while every thread works at
623        least 3s.  So without releasing GIL test should fail.
624        """
625
626        self.load('threading')
627
628        for _ in range(10):
629            self.get(no_recv=True)
630
631        self.assertIsNotNone(
632            self.wait_for_record(r'\(5\) Thread: 100'), 'last thread finished'
633        )
634
635    def test_python_application_iter_exception(self):
636        self.load('iter_exception')
637
638        # Default request doesn't lead to the exception.
639
640        resp = self.get(
641            headers={
642                'Host': 'localhost',
643                'X-Skip': '9',
644                'X-Chunked': '1',
645                'Connection': 'close',
646            }
647        )
648        self.assertEqual(resp['status'], 200, 'status')
649        self.assertEqual(resp['body'], 'XXXXXXX', 'body')
650
651        # Exception before start_response().
652
653        self.assertEqual(self.get()['status'], 503, 'error')
654
655        self.assertIsNotNone(self.wait_for_record(r'Traceback'), 'traceback')
656        self.assertIsNotNone(
657            self.wait_for_record(r'raise Exception\(\'first exception\'\)'),
658            'first exception raise',
659        )
660        self.assertEqual(
661            len(self.findall(r'Traceback')), 1, 'traceback count 1'
662        )
663
664        # Exception after start_response(), before first write().
665
666        self.assertEqual(
667            self.get(
668                headers={
669                    'Host': 'localhost',
670                    'X-Skip': '1',
671                    'Connection': 'close',
672                }
673            )['status'],
674            503,
675            'error 2',
676        )
677
678        self.assertIsNotNone(
679            self.wait_for_record(r'raise Exception\(\'second exception\'\)'),
680            'exception raise second',
681        )
682        self.assertEqual(
683            len(self.findall(r'Traceback')), 2, 'traceback count 2'
684        )
685
686        # Exception after first write(), before first __next__().
687
688        _, sock = self.get(
689            headers={
690                'Host': 'localhost',
691                'X-Skip': '2',
692                'Connection': 'keep-alive',
693            },
694            start=True,
695        )
696
697        self.assertIsNotNone(
698            self.wait_for_record(r'raise Exception\(\'third exception\'\)'),
699            'exception raise third',
700        )
701        self.assertEqual(
702            len(self.findall(r'Traceback')), 3, 'traceback count 3'
703        )
704
705        self.assertDictEqual(self.get(sock=sock), {}, 'closed connection')
706
707        # Exception after first write(), before first __next__(),
708        # chunked (incomplete body).
709
710        resp = self.get(
711            headers={
712                'Host': 'localhost',
713                'X-Skip': '2',
714                'X-Chunked': '1',
715                'Connection': 'close',
716            },
717            raw_resp=True
718        )
719        if resp:
720            self.assertNotEqual(resp[-5:], '0\r\n\r\n', 'incomplete body')
721        self.assertEqual(
722            len(self.findall(r'Traceback')), 4, 'traceback count 4'
723        )
724
725        # Exception in __next__().
726
727        _, sock = self.get(
728            headers={
729                'Host': 'localhost',
730                'X-Skip': '3',
731                'Connection': 'keep-alive',
732            },
733            start=True,
734        )
735
736        self.assertIsNotNone(
737            self.wait_for_record(r'raise Exception\(\'next exception\'\)'),
738            'exception raise next',
739        )
740        self.assertEqual(
741            len(self.findall(r'Traceback')), 5, 'traceback count 5'
742        )
743
744        self.assertDictEqual(self.get(sock=sock), {}, 'closed connection 2')
745
746        # Exception in __next__(), chunked (incomplete body).
747
748        resp = self.get(
749            headers={
750                'Host': 'localhost',
751                'X-Skip': '3',
752                'X-Chunked': '1',
753                'Connection': 'close',
754            },
755            raw_resp=True
756        )
757        if resp:
758            self.assertNotEqual(resp[-5:], '0\r\n\r\n', 'incomplete body 2')
759        self.assertEqual(
760            len(self.findall(r'Traceback')), 6, 'traceback count 6'
761        )
762
763        # Exception before start_response() and in close().
764
765        self.assertEqual(
766            self.get(
767                headers={
768                    'Host': 'localhost',
769                    'X-Not-Skip-Close': '1',
770                    'Connection': 'close',
771                }
772            )['status'],
773            503,
774            'error',
775        )
776
777        self.assertIsNotNone(
778            self.wait_for_record(r'raise Exception\(\'close exception\'\)'),
779            'exception raise close',
780        )
781        self.assertEqual(
782            len(self.findall(r'Traceback')), 8, 'traceback count 8'
783        )
784
785    def test_python_user_group(self):
786        if not self.is_su:
787            print("requires root")
788            raise unittest.SkipTest()
789
790        nobody_uid = pwd.getpwnam('nobody').pw_uid
791
792        group = 'nobody'
793
794        try:
795            group_id = grp.getgrnam(group).gr_gid
796        except:
797            group = 'nogroup'
798            group_id = grp.getgrnam(group).gr_gid
799
800        self.load('user_group')
801
802        obj = self.getjson()['body']
803        self.assertEqual(obj['UID'], nobody_uid, 'nobody uid')
804        self.assertEqual(obj['GID'], group_id, 'nobody gid')
805
806        self.load('user_group', user='nobody')
807
808        obj = self.getjson()['body']
809        self.assertEqual(obj['UID'], nobody_uid, 'nobody uid user=nobody')
810        self.assertEqual(obj['GID'], group_id, 'nobody gid user=nobody')
811
812        self.load('user_group', user='nobody', group=group)
813
814        obj = self.getjson()['body']
815        self.assertEqual(
816            obj['UID'], nobody_uid, 'nobody uid user=nobody group=%s' % group
817        )
818
819        self.assertEqual(
820            obj['GID'], group_id, 'nobody gid user=nobody group=%s' % group
821        )
822
823        self.load('user_group', group=group)
824
825        obj = self.getjson()['body']
826        self.assertEqual(
827            obj['UID'], nobody_uid, 'nobody uid group=%s' % group
828        )
829
830        self.assertEqual(obj['GID'], group_id, 'nobody gid group=%s' % group)
831
832        self.load('user_group', user='root')
833
834        obj = self.getjson()['body']
835        self.assertEqual(obj['UID'], 0, 'root uid user=root')
836        self.assertEqual(obj['GID'], 0, 'root gid user=root')
837
838        group = 'root'
839
840        try:
841            grp.getgrnam(group)
842            group = True
843        except:
844            group = False
845
846        if group:
847            self.load('user_group', user='root', group='root')
848
849            obj = self.getjson()['body']
850            self.assertEqual(obj['UID'], 0, 'root uid user=root group=root')
851            self.assertEqual(obj['GID'], 0, 'root gid user=root group=root')
852
853            self.load('user_group', group='root')
854
855            obj = self.getjson()['body']
856            self.assertEqual(obj['UID'], nobody_uid, 'root uid group=root')
857            self.assertEqual(obj['GID'], 0, 'root gid group=root')
858
859if __name__ == '__main__':
860    TestPythonApplication.main()
861