xref: /unit/test/test_python_application.py (revision 1568:b0df4227285f)
1import grp
2import pwd
3import re
4import time
5import unittest
6
7from unit.applications.lang.python import TestApplicationPython
8
9
10class TestPythonApplication(TestApplicationPython):
11    prerequisites = {'modules': {'python': 'all'}}
12
13    def findall(self, pattern):
14        with open(self.testdir + '/unit.log', 'r', errors='ignore') as f:
15            return re.findall(pattern, f.read())
16
17    def test_python_application_variables(self):
18        self.load('variables')
19
20        body = 'Test body string.'
21
22        resp = self.post(
23            headers={
24                'Host': 'localhost',
25                'Content-Type': 'text/html',
26                'Custom-Header': 'blah',
27                'Connection': 'close',
28            },
29            body=body,
30        )
31
32        self.assertEqual(resp['status'], 200, 'status')
33        headers = resp['headers']
34        header_server = headers.pop('Server')
35        self.assertRegex(header_server, r'Unit/[\d\.]+', 'server header')
36        self.assertEqual(
37            headers.pop('Server-Software'),
38            header_server,
39            'server software header',
40        )
41
42        date = headers.pop('Date')
43        self.assertEqual(date[-4:], ' GMT', 'date header timezone')
44        self.assertLess(
45            abs(self.date_to_sec_epoch(date) - self.sec_epoch()),
46            5,
47            'date header',
48        )
49
50        self.assertDictEqual(
51            headers,
52            {
53                'Connection': 'close',
54                'Content-Length': str(len(body)),
55                'Content-Type': 'text/html',
56                'Request-Method': 'POST',
57                'Request-Uri': '/',
58                'Http-Host': 'localhost',
59                'Server-Protocol': 'HTTP/1.1',
60                'Custom-Header': 'blah',
61                'Wsgi-Version': '(1, 0)',
62                'Wsgi-Url-Scheme': 'http',
63                'Wsgi-Multithread': 'False',
64                'Wsgi-Multiprocess': 'True',
65                'Wsgi-Run-Once': 'False',
66            },
67            'headers',
68        )
69        self.assertEqual(resp['body'], body, 'body')
70
71    def test_python_application_query_string(self):
72        self.load('query_string')
73
74        resp = self.get(url='/?var1=val1&var2=val2')
75
76        self.assertEqual(
77            resp['headers']['Query-String'],
78            'var1=val1&var2=val2',
79            'Query-String header',
80        )
81
82    def test_python_application_query_string_space(self):
83        self.load('query_string')
84
85        resp = self.get(url='/ ?var1=val1&var2=val2')
86        self.assertEqual(
87            resp['headers']['Query-String'],
88            'var1=val1&var2=val2',
89            'Query-String space',
90        )
91
92        resp = self.get(url='/ %20?var1=val1&var2=val2')
93        self.assertEqual(
94            resp['headers']['Query-String'],
95            'var1=val1&var2=val2',
96            'Query-String space 2',
97        )
98
99        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
100        self.assertEqual(
101            resp['headers']['Query-String'],
102            'var1=val1&var2=val2',
103            'Query-String space 3',
104        )
105
106        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
107        self.assertEqual(
108            resp['headers']['Query-String'],
109            ' var1= val1 & var2=val2',
110            'Query-String space 4',
111        )
112
113    def test_python_application_query_string_empty(self):
114        self.load('query_string')
115
116        resp = self.get(url='/?')
117
118        self.assertEqual(resp['status'], 200, 'query string empty status')
119        self.assertEqual(
120            resp['headers']['Query-String'], '', 'query string empty'
121        )
122
123    def test_python_application_query_string_absent(self):
124        self.load('query_string')
125
126        resp = self.get()
127
128        self.assertEqual(resp['status'], 200, 'query string absent status')
129        self.assertEqual(
130            resp['headers']['Query-String'], '', 'query string absent'
131        )
132
133    @unittest.skip('not yet')
134    def test_python_application_server_port(self):
135        self.load('server_port')
136
137        self.assertEqual(
138            self.get()['headers']['Server-Port'], '7080', 'Server-Port header'
139        )
140
141    @unittest.skip('not yet')
142    def test_python_application_working_directory_invalid(self):
143        self.load('empty')
144
145        self.assertIn(
146            'success',
147            self.conf('"/blah"', 'applications/empty/working_directory'),
148            'configure invalid working_directory',
149        )
150
151        self.assertEqual(self.get()['status'], 500, 'status')
152
153    def test_python_application_204_transfer_encoding(self):
154        self.load('204_no_content')
155
156        self.assertNotIn(
157            'Transfer-Encoding',
158            self.get()['headers'],
159            '204 header transfer encoding',
160        )
161
162    def test_python_application_ctx_iter_atexit(self):
163        self.load('ctx_iter_atexit')
164
165        resp = self.post(
166            headers={
167                'Host': 'localhost',
168                'Connection': 'close',
169                'Content-Type': 'text/html',
170            },
171            body='0123456789',
172        )
173
174        self.assertEqual(resp['status'], 200, 'ctx iter status')
175        self.assertEqual(resp['body'], '0123456789', 'ctx iter body')
176
177        self.conf({"listeners": {}, "applications": {}})
178
179        self.stop()
180
181        self.assertIsNotNone(
182            self.wait_for_record(r'RuntimeError'), 'ctx iter atexit'
183        )
184
185    def test_python_keepalive_body(self):
186        self.load('mirror')
187
188        self.assertEqual(self.get()['status'], 200, 'init')
189
190        body = '0123456789' * 500
191        (resp, sock) = self.post(
192            headers={
193                'Host': 'localhost',
194                'Connection': 'keep-alive',
195                'Content-Type': 'text/html',
196            },
197            start=True,
198            body=body,
199            read_timeout=1,
200        )
201
202        self.assertEqual(resp['body'], body, 'keep-alive 1')
203
204        body = '0123456789'
205        resp = self.post(
206            headers={
207                'Host': 'localhost',
208                'Connection': 'close',
209                'Content-Type': 'text/html',
210            },
211            sock=sock,
212            body=body,
213        )
214
215        self.assertEqual(resp['body'], body, 'keep-alive 2')
216
217    def test_python_keepalive_reconfigure(self):
218        self.skip_alerts.extend(
219            [
220                r'pthread_mutex.+failed',
221                r'failed to apply',
222                r'process \d+ exited on signal',
223            ]
224        )
225        self.load('mirror')
226
227        self.assertEqual(self.get()['status'], 200, 'init')
228
229        body = '0123456789'
230        conns = 3
231        socks = []
232
233        for i in range(conns):
234            (resp, sock) = self.post(
235                headers={
236                    'Host': 'localhost',
237                    'Connection': 'keep-alive',
238                    'Content-Type': 'text/html',
239                },
240                start=True,
241                body=body,
242                read_timeout=1,
243            )
244
245            self.assertEqual(resp['body'], body, 'keep-alive open')
246            self.assertIn(
247                'success',
248                self.conf(str(i + 1), 'applications/mirror/processes'),
249                'reconfigure',
250            )
251
252            socks.append(sock)
253
254        for i in range(conns):
255            (resp, sock) = self.post(
256                headers={
257                    'Host': 'localhost',
258                    'Connection': 'keep-alive',
259                    'Content-Type': 'text/html',
260                },
261                start=True,
262                sock=socks[i],
263                body=body,
264                read_timeout=1,
265            )
266
267            self.assertEqual(resp['body'], body, 'keep-alive request')
268            self.assertIn(
269                'success',
270                self.conf(str(i + 1), 'applications/mirror/processes'),
271                'reconfigure 2',
272            )
273
274        for i in range(conns):
275            resp = self.post(
276                headers={
277                    'Host': 'localhost',
278                    'Connection': 'close',
279                    'Content-Type': 'text/html',
280                },
281                sock=socks[i],
282                body=body,
283            )
284
285            self.assertEqual(resp['body'], body, 'keep-alive close')
286            self.assertIn(
287                'success',
288                self.conf(str(i + 1), 'applications/mirror/processes'),
289                'reconfigure 3',
290            )
291
292    def test_python_keepalive_reconfigure_2(self):
293        self.load('mirror')
294
295        self.assertEqual(self.get()['status'], 200, 'init')
296
297        body = '0123456789'
298
299        (resp, sock) = self.post(
300            headers={
301                'Host': 'localhost',
302                'Connection': 'keep-alive',
303                'Content-Type': 'text/html',
304            },
305            start=True,
306            body=body,
307            read_timeout=1,
308        )
309
310        self.assertEqual(resp['body'], body, 'reconfigure 2 keep-alive 1')
311
312        self.load('empty')
313
314        self.assertEqual(self.get()['status'], 200, 'init')
315
316        (resp, sock) = self.post(
317            headers={
318                'Host': 'localhost',
319                'Connection': 'close',
320                'Content-Type': 'text/html',
321            },
322            start=True,
323            sock=sock,
324            body=body,
325        )
326
327        self.assertEqual(resp['status'], 200, 'reconfigure 2 keep-alive 2')
328        self.assertEqual(resp['body'], '', 'reconfigure 2 keep-alive 2 body')
329
330        self.assertIn(
331            'success',
332            self.conf({"listeners": {}, "applications": {}}),
333            'reconfigure 2 clear configuration',
334        )
335
336        resp = self.get(sock=sock)
337
338        self.assertEqual(resp, {}, 'reconfigure 2 keep-alive 3')
339
340    def test_python_keepalive_reconfigure_3(self):
341        self.load('empty')
342
343        self.assertEqual(self.get()['status'], 200, 'init')
344
345        (_, sock) = self.http(
346            b"""GET / HTTP/1.1
347""",
348            start=True,
349            raw=True,
350            no_recv=True,
351        )
352
353        self.assertEqual(self.get()['status'], 200)
354
355        self.assertIn(
356            'success',
357            self.conf({"listeners": {}, "applications": {}}),
358            'reconfigure 3 clear configuration',
359        )
360
361        resp = self.http(
362            b"""Host: localhost
363Connection: close
364
365""",
366            sock=sock,
367            raw=True,
368        )
369
370        self.assertEqual(resp['status'], 200, 'reconfigure 3')
371
372    def test_python_atexit(self):
373        self.load('atexit')
374
375        self.get()
376
377        self.conf({"listeners": {}, "applications": {}})
378
379        self.stop()
380
381        self.assertIsNotNone(
382            self.wait_for_record(r'At exit called\.'), 'atexit'
383        )
384
385    def test_python_process_switch(self):
386        self.load('delayed')
387
388        self.assertIn(
389            'success',
390            self.conf('2', 'applications/delayed/processes'),
391            'configure 2 processes',
392        )
393
394        self.get(headers={
395            'Host': 'localhost',
396            'Content-Length': '0',
397            'X-Delay': '5',
398            'Connection': 'close',
399        }, no_recv=True)
400
401        headers_delay_1 = {
402            'Connection': 'close',
403            'Host': 'localhost',
404            'Content-Length': '0',
405            'X-Delay': '1',
406        }
407
408        self.get(headers=headers_delay_1, no_recv=True)
409
410        time.sleep(0.5)
411
412        for _ in range(10):
413            self.get(headers=headers_delay_1, no_recv=True)
414
415        self.get(headers=headers_delay_1)
416
417    @unittest.skip('not yet')
418    def test_python_application_start_response_exit(self):
419        self.load('start_response_exit')
420
421        self.assertEqual(self.get()['status'], 500, 'start response exit')
422
423    def test_python_application_input_iter(self):
424        self.load('input_iter')
425
426        body = '''0123456789
427next line
428
429last line'''
430
431        resp = self.post(body=body)
432        self.assertEqual(resp['body'], body, 'input iter')
433        self.assertEqual(
434            resp['headers']['X-Lines-Count'], '4', 'input iter lines'
435        )
436
437    def test_python_application_input_readline(self):
438        self.load('input_readline')
439
440        body = '''0123456789
441next line
442
443last line'''
444
445        resp = self.post(body=body)
446        self.assertEqual(resp['body'], body, 'input readline')
447        self.assertEqual(
448            resp['headers']['X-Lines-Count'], '4', 'input readline lines'
449        )
450
451    def test_python_application_input_readline_size(self):
452        self.load('input_readline_size')
453
454        body = '''0123456789
455next line
456
457last line'''
458
459        self.assertEqual(
460            self.post(body=body)['body'], body, 'input readline size'
461        )
462        self.assertEqual(
463            self.post(body='0123')['body'], '0123', 'input readline size less'
464        )
465
466    def test_python_application_input_readlines(self):
467        self.load('input_readlines')
468
469        body = '''0123456789
470next line
471
472last line'''
473
474        resp = self.post(body=body)
475        self.assertEqual(resp['body'], body, 'input readlines')
476        self.assertEqual(
477            resp['headers']['X-Lines-Count'], '4', 'input readlines lines'
478        )
479
480    def test_python_application_input_readlines_huge(self):
481        self.load('input_readlines')
482
483        body = (
484            '''0123456789 abcdefghi
485next line: 0123456789 abcdefghi
486
487last line: 987654321
488'''
489            * 512
490        )
491
492        self.assertEqual(
493            self.post(body=body, read_buffer_size=16384)['body'],
494            body,
495            'input readlines huge',
496        )
497
498    def test_python_application_input_read_length(self):
499        self.load('input_read_length')
500
501        body = '0123456789'
502
503        resp = self.post(
504            headers={
505                'Host': 'localhost',
506                'Input-Length': '5',
507                'Connection': 'close',
508            },
509            body=body,
510        )
511
512        self.assertEqual(resp['body'], body[:5], 'input read length lt body')
513
514        resp = self.post(
515            headers={
516                'Host': 'localhost',
517                'Input-Length': '15',
518                'Connection': 'close',
519            },
520            body=body,
521        )
522
523        self.assertEqual(resp['body'], body, 'input read length gt body')
524
525        resp = self.post(
526            headers={
527                'Host': 'localhost',
528                'Input-Length': '0',
529                'Connection': 'close',
530            },
531            body=body,
532        )
533
534        self.assertEqual(resp['body'], '', 'input read length zero')
535
536        resp = self.post(
537            headers={
538                'Host': 'localhost',
539                'Input-Length': '-1',
540                'Connection': 'close',
541            },
542            body=body,
543        )
544
545        self.assertEqual(resp['body'], body, 'input read length negative')
546
547    @unittest.skip('not yet')
548    def test_python_application_errors_write(self):
549        self.load('errors_write')
550
551        self.get()
552
553        self.stop()
554
555        self.assertIsNotNone(
556            self.wait_for_record(r'\[error\].+Error in application\.'),
557            'errors write',
558        )
559
560    def test_python_application_body_array(self):
561        self.load('body_array')
562
563        self.assertEqual(self.get()['body'], '0123456789', 'body array')
564
565    def test_python_application_body_io(self):
566        self.load('body_io')
567
568        self.assertEqual(self.get()['body'], '0123456789', 'body io')
569
570    def test_python_application_body_io_file(self):
571        self.load('body_io_file')
572
573        self.assertEqual(self.get()['body'], 'body\n', 'body io file')
574
575    @unittest.skip('not yet')
576    def test_python_application_syntax_error(self):
577        self.skip_alerts.append(r'Python failed to import module "wsgi"')
578        self.load('syntax_error')
579
580        self.assertEqual(self.get()['status'], 500, 'syntax error')
581
582    def test_python_application_loading_error(self):
583        self.skip_alerts.append(r'Python failed to import module "blah"')
584
585        self.load('empty')
586
587        self.assertIn(
588            'success', self.conf('"blah"', 'applications/empty/module'),
589        )
590
591        self.assertEqual(self.get()['status'], 503, 'loading error')
592
593    def test_python_application_close(self):
594        self.load('close')
595
596        self.get()
597
598        self.stop()
599
600        self.assertIsNotNone(self.wait_for_record(r'Close called\.'), 'close')
601
602    def test_python_application_close_error(self):
603        self.load('close_error')
604
605        self.get()
606
607        self.stop()
608
609        self.assertIsNotNone(
610            self.wait_for_record(r'Close called\.'), 'close error'
611        )
612
613    def test_python_application_not_iterable(self):
614        self.load('not_iterable')
615
616        self.get()
617
618        self.stop()
619
620        self.assertIsNotNone(
621            self.wait_for_record(
622                r'\[error\].+the application returned not an iterable object'
623            ),
624            'not iterable',
625        )
626
627    def test_python_application_write(self):
628        self.load('write')
629
630        self.assertEqual(self.get()['body'], '0123456789', 'write')
631
632    def test_python_application_threading(self):
633        """wait_for_record() timeouts after 5s while every thread works at
634        least 3s.  So without releasing GIL test should fail.
635        """
636
637        self.load('threading')
638
639        for _ in range(10):
640            self.get(no_recv=True)
641
642        self.assertIsNotNone(
643            self.wait_for_record(r'\(5\) Thread: 100'), 'last thread finished'
644        )
645
646    def test_python_application_iter_exception(self):
647        self.load('iter_exception')
648
649        # Default request doesn't lead to the exception.
650
651        resp = self.get(
652            headers={
653                'Host': 'localhost',
654                'X-Skip': '9',
655                'X-Chunked': '1',
656                'Connection': 'close',
657            }
658        )
659        self.assertEqual(resp['status'], 200, 'status')
660        self.assertEqual(resp['body'], 'XXXXXXX', 'body')
661
662        # Exception before start_response().
663
664        self.assertEqual(self.get()['status'], 503, 'error')
665
666        self.assertIsNotNone(self.wait_for_record(r'Traceback'), 'traceback')
667        self.assertIsNotNone(
668            self.wait_for_record(r'raise Exception\(\'first exception\'\)'),
669            'first exception raise',
670        )
671        self.assertEqual(
672            len(self.findall(r'Traceback')), 1, 'traceback count 1'
673        )
674
675        # Exception after start_response(), before first write().
676
677        self.assertEqual(
678            self.get(
679                headers={
680                    'Host': 'localhost',
681                    'X-Skip': '1',
682                    'Connection': 'close',
683                }
684            )['status'],
685            503,
686            'error 2',
687        )
688
689        self.assertIsNotNone(
690            self.wait_for_record(r'raise Exception\(\'second exception\'\)'),
691            'exception raise second',
692        )
693        self.assertEqual(
694            len(self.findall(r'Traceback')), 2, 'traceback count 2'
695        )
696
697        # Exception after first write(), before first __next__().
698
699        _, sock = self.get(
700            headers={
701                'Host': 'localhost',
702                'X-Skip': '2',
703                'Connection': 'keep-alive',
704            },
705            start=True,
706        )
707
708        self.assertIsNotNone(
709            self.wait_for_record(r'raise Exception\(\'third exception\'\)'),
710            'exception raise third',
711        )
712        self.assertEqual(
713            len(self.findall(r'Traceback')), 3, 'traceback count 3'
714        )
715
716        self.assertDictEqual(self.get(sock=sock), {}, 'closed connection')
717
718        # Exception after first write(), before first __next__(),
719        # chunked (incomplete body).
720
721        resp = self.get(
722            headers={
723                'Host': 'localhost',
724                'X-Skip': '2',
725                'X-Chunked': '1',
726                'Connection': 'close',
727            },
728            raw_resp=True
729        )
730        if resp:
731            self.assertNotEqual(resp[-5:], '0\r\n\r\n', 'incomplete body')
732        self.assertEqual(
733            len(self.findall(r'Traceback')), 4, 'traceback count 4'
734        )
735
736        # Exception in __next__().
737
738        _, sock = self.get(
739            headers={
740                'Host': 'localhost',
741                'X-Skip': '3',
742                'Connection': 'keep-alive',
743            },
744            start=True,
745        )
746
747        self.assertIsNotNone(
748            self.wait_for_record(r'raise Exception\(\'next exception\'\)'),
749            'exception raise next',
750        )
751        self.assertEqual(
752            len(self.findall(r'Traceback')), 5, 'traceback count 5'
753        )
754
755        self.assertDictEqual(self.get(sock=sock), {}, 'closed connection 2')
756
757        # Exception in __next__(), chunked (incomplete body).
758
759        resp = self.get(
760            headers={
761                'Host': 'localhost',
762                'X-Skip': '3',
763                'X-Chunked': '1',
764                'Connection': 'close',
765            },
766            raw_resp=True
767        )
768        if resp:
769            self.assertNotEqual(resp[-5:], '0\r\n\r\n', 'incomplete body 2')
770        self.assertEqual(
771            len(self.findall(r'Traceback')), 6, 'traceback count 6'
772        )
773
774        # Exception before start_response() and in close().
775
776        self.assertEqual(
777            self.get(
778                headers={
779                    'Host': 'localhost',
780                    'X-Not-Skip-Close': '1',
781                    'Connection': 'close',
782                }
783            )['status'],
784            503,
785            'error',
786        )
787
788        self.assertIsNotNone(
789            self.wait_for_record(r'raise Exception\(\'close exception\'\)'),
790            'exception raise close',
791        )
792        self.assertEqual(
793            len(self.findall(r'Traceback')), 8, 'traceback count 8'
794        )
795
796    def test_python_user_group(self):
797        if not self.is_su:
798            print("requires root")
799            raise unittest.SkipTest()
800
801        nobody_uid = pwd.getpwnam('nobody').pw_uid
802
803        group = 'nobody'
804
805        try:
806            group_id = grp.getgrnam(group).gr_gid
807        except:
808            group = 'nogroup'
809            group_id = grp.getgrnam(group).gr_gid
810
811        self.load('user_group')
812
813        obj = self.getjson()['body']
814        self.assertEqual(obj['UID'], nobody_uid, 'nobody uid')
815        self.assertEqual(obj['GID'], group_id, 'nobody gid')
816
817        self.load('user_group', user='nobody')
818
819        obj = self.getjson()['body']
820        self.assertEqual(obj['UID'], nobody_uid, 'nobody uid user=nobody')
821        self.assertEqual(obj['GID'], group_id, 'nobody gid user=nobody')
822
823        self.load('user_group', user='nobody', group=group)
824
825        obj = self.getjson()['body']
826        self.assertEqual(
827            obj['UID'], nobody_uid, 'nobody uid user=nobody group=%s' % group
828        )
829
830        self.assertEqual(
831            obj['GID'], group_id, 'nobody gid user=nobody group=%s' % group
832        )
833
834        self.load('user_group', group=group)
835
836        obj = self.getjson()['body']
837        self.assertEqual(
838            obj['UID'], nobody_uid, 'nobody uid group=%s' % group
839        )
840
841        self.assertEqual(obj['GID'], group_id, 'nobody gid group=%s' % group)
842
843        self.load('user_group', user='root')
844
845        obj = self.getjson()['body']
846        self.assertEqual(obj['UID'], 0, 'root uid user=root')
847        self.assertEqual(obj['GID'], 0, 'root gid user=root')
848
849        group = 'root'
850
851        try:
852            grp.getgrnam(group)
853            group = True
854        except:
855            group = False
856
857        if group:
858            self.load('user_group', user='root', group='root')
859
860            obj = self.getjson()['body']
861            self.assertEqual(obj['UID'], 0, 'root uid user=root group=root')
862            self.assertEqual(obj['GID'], 0, 'root gid user=root group=root')
863
864            self.load('user_group', group='root')
865
866            obj = self.getjson()['body']
867            self.assertEqual(obj['UID'], nobody_uid, 'root uid group=root')
868            self.assertEqual(obj['GID'], 0, 'root gid group=root')
869
870if __name__ == '__main__':
871    TestPythonApplication.main()
872