xref: /unit/test/test_python_application.py (revision 1453:71af60a59338)
1import re
2import os
3import grp
4import pwd
5import time
6import unittest
7from unit.applications.lang.python import TestApplicationPython
8
9
10class TestPythonApplication(TestApplicationPython):
11    prerequisites = {'modules': ['python']}
12
13    def findall(self, pattern):
14        with open(self.testdir + '/unit.log', 'r', errors='ignore') as f:
15            return re.findall(pattern, f.read())
16
17    def test_python_application_variables(self):
18        self.load('variables')
19
20        body = 'Test body string.'
21
22        resp = self.post(
23            headers={
24                'Host': 'localhost',
25                'Content-Type': 'text/html',
26                'Custom-Header': 'blah',
27                'Connection': 'close',
28            },
29            body=body,
30        )
31
32        self.assertEqual(resp['status'], 200, 'status')
33        headers = resp['headers']
34        header_server = headers.pop('Server')
35        self.assertRegex(header_server, r'Unit/[\d\.]+', 'server header')
36        self.assertEqual(
37            headers.pop('Server-Software'),
38            header_server,
39            'server software header',
40        )
41
42        date = headers.pop('Date')
43        self.assertEqual(date[-4:], ' GMT', 'date header timezone')
44        self.assertLess(
45            abs(self.date_to_sec_epoch(date) - self.sec_epoch()),
46            5,
47            'date header',
48        )
49
50        self.assertDictEqual(
51            headers,
52            {
53                'Connection': 'close',
54                'Content-Length': str(len(body)),
55                'Content-Type': 'text/html',
56                'Request-Method': 'POST',
57                'Request-Uri': '/',
58                'Http-Host': 'localhost',
59                'Server-Protocol': 'HTTP/1.1',
60                'Custom-Header': 'blah',
61                'Wsgi-Version': '(1, 0)',
62                'Wsgi-Url-Scheme': 'http',
63                'Wsgi-Multithread': 'False',
64                'Wsgi-Multiprocess': 'True',
65                'Wsgi-Run-Once': 'False',
66            },
67            'headers',
68        )
69        self.assertEqual(resp['body'], body, 'body')
70
71    def test_python_application_query_string(self):
72        self.load('query_string')
73
74        resp = self.get(url='/?var1=val1&var2=val2')
75
76        self.assertEqual(
77            resp['headers']['Query-String'],
78            'var1=val1&var2=val2',
79            'Query-String header',
80        )
81
82    def test_python_application_query_string_space(self):
83        self.load('query_string')
84
85        resp = self.get(url='/ ?var1=val1&var2=val2')
86        self.assertEqual(
87            resp['headers']['Query-String'],
88            'var1=val1&var2=val2',
89            'Query-String space',
90        )
91
92        resp = self.get(url='/ %20?var1=val1&var2=val2')
93        self.assertEqual(
94            resp['headers']['Query-String'],
95            'var1=val1&var2=val2',
96            'Query-String space 2',
97        )
98
99        resp = self.get(url='/ %20 ?var1=val1&var2=val2')
100        self.assertEqual(
101            resp['headers']['Query-String'],
102            'var1=val1&var2=val2',
103            'Query-String space 3',
104        )
105
106        resp = self.get(url='/blah %20 blah? var1= val1 & var2=val2')
107        self.assertEqual(
108            resp['headers']['Query-String'],
109            ' var1= val1 & var2=val2',
110            'Query-String space 4',
111        )
112
113    def test_python_application_query_string_empty(self):
114        self.load('query_string')
115
116        resp = self.get(url='/?')
117
118        self.assertEqual(resp['status'], 200, 'query string empty status')
119        self.assertEqual(
120            resp['headers']['Query-String'], '', 'query string empty'
121        )
122
123    def test_python_application_query_string_absent(self):
124        self.load('query_string')
125
126        resp = self.get()
127
128        self.assertEqual(resp['status'], 200, 'query string absent status')
129        self.assertEqual(
130            resp['headers']['Query-String'], '', 'query string absent'
131        )
132
133    @unittest.skip('not yet')
134    def test_python_application_server_port(self):
135        self.load('server_port')
136
137        self.assertEqual(
138            self.get()['headers']['Server-Port'], '7080', 'Server-Port header'
139        )
140
141    @unittest.skip('not yet')
142    def test_python_application_working_directory_invalid(self):
143        self.load('empty')
144
145        self.assertIn(
146            'success',
147            self.conf('"/blah"', 'applications/empty/working_directory'),
148            'configure invalid working_directory',
149        )
150
151        self.assertEqual(self.get()['status'], 500, 'status')
152
153    def test_python_application_204_transfer_encoding(self):
154        self.load('204_no_content')
155
156        self.assertNotIn(
157            'Transfer-Encoding',
158            self.get()['headers'],
159            '204 header transfer encoding',
160        )
161
162    def test_python_application_ctx_iter_atexit(self):
163        self.load('ctx_iter_atexit')
164
165        resp = self.post(
166            headers={
167                'Host': 'localhost',
168                'Connection': 'close',
169                'Content-Type': 'text/html',
170            },
171            body='0123456789',
172        )
173
174        self.assertEqual(resp['status'], 200, 'ctx iter status')
175        self.assertEqual(resp['body'], '0123456789', 'ctx iter body')
176
177        self.conf({"listeners": {}, "applications": {}})
178
179        self.stop()
180
181        self.assertIsNotNone(
182            self.wait_for_record(r'RuntimeError'), 'ctx iter atexit'
183        )
184
185    def test_python_keepalive_body(self):
186        self.load('mirror')
187
188        self.assertEqual(self.get()['status'], 200, 'init')
189
190        body = '0123456789' * 500
191        (resp, sock) = self.post(
192            headers={
193                'Host': 'localhost',
194                'Connection': 'keep-alive',
195                'Content-Type': 'text/html',
196            },
197            start=True,
198            body=body,
199            read_timeout=1,
200        )
201
202        self.assertEqual(resp['body'], body, 'keep-alive 1')
203
204        body = '0123456789'
205        resp = self.post(
206            headers={
207                'Host': 'localhost',
208                'Connection': 'close',
209                'Content-Type': 'text/html',
210            },
211            sock=sock,
212            body=body,
213        )
214
215        self.assertEqual(resp['body'], body, 'keep-alive 2')
216
217    def test_python_keepalive_reconfigure(self):
218        self.skip_alerts.extend(
219            [
220                r'pthread_mutex.+failed',
221                r'failed to apply',
222                r'process \d+ exited on signal',
223            ]
224        )
225        self.load('mirror')
226
227        self.assertEqual(self.get()['status'], 200, 'init')
228
229        body = '0123456789'
230        conns = 3
231        socks = []
232
233        for i in range(conns):
234            (resp, sock) = self.post(
235                headers={
236                    'Host': 'localhost',
237                    'Connection': 'keep-alive',
238                    'Content-Type': 'text/html',
239                },
240                start=True,
241                body=body,
242                read_timeout=1,
243            )
244
245            self.assertEqual(resp['body'], body, 'keep-alive open')
246            self.assertIn(
247                'success',
248                self.conf(str(i + 1), 'applications/mirror/processes'),
249                'reconfigure',
250            )
251
252            socks.append(sock)
253
254        for i in range(conns):
255            (resp, sock) = self.post(
256                headers={
257                    'Host': 'localhost',
258                    'Connection': 'keep-alive',
259                    'Content-Type': 'text/html',
260                },
261                start=True,
262                sock=socks[i],
263                body=body,
264                read_timeout=1,
265            )
266
267            self.assertEqual(resp['body'], body, 'keep-alive request')
268            self.assertIn(
269                'success',
270                self.conf(str(i + 1), 'applications/mirror/processes'),
271                'reconfigure 2',
272            )
273
274        for i in range(conns):
275            resp = self.post(
276                headers={
277                    'Host': 'localhost',
278                    'Connection': 'close',
279                    'Content-Type': 'text/html',
280                },
281                sock=socks[i],
282                body=body,
283            )
284
285            self.assertEqual(resp['body'], body, 'keep-alive close')
286            self.assertIn(
287                'success',
288                self.conf(str(i + 1), 'applications/mirror/processes'),
289                'reconfigure 3',
290            )
291
292    def test_python_keepalive_reconfigure_2(self):
293        self.load('mirror')
294
295        self.assertEqual(self.get()['status'], 200, 'init')
296
297        body = '0123456789'
298
299        (resp, sock) = self.post(
300            headers={
301                'Host': 'localhost',
302                'Connection': 'keep-alive',
303                'Content-Type': 'text/html',
304            },
305            start=True,
306            body=body,
307            read_timeout=1,
308        )
309
310        self.assertEqual(resp['body'], body, 'reconfigure 2 keep-alive 1')
311
312        self.load('empty')
313
314        self.assertEqual(self.get()['status'], 200, 'init')
315
316        (resp, sock) = self.post(
317            headers={
318                'Host': 'localhost',
319                'Connection': 'close',
320                'Content-Type': 'text/html',
321            },
322            start=True,
323            sock=sock,
324            body=body,
325        )
326
327        self.assertEqual(resp['status'], 200, 'reconfigure 2 keep-alive 2')
328        self.assertEqual(resp['body'], '', 'reconfigure 2 keep-alive 2 body')
329
330        self.assertIn(
331            'success',
332            self.conf({"listeners": {}, "applications": {}}),
333            'reconfigure 2 clear configuration',
334        )
335
336        resp = self.get(sock=sock)
337
338        self.assertEqual(resp, {}, 'reconfigure 2 keep-alive 3')
339
340    def test_python_keepalive_reconfigure_3(self):
341        self.load('empty')
342
343        self.assertEqual(self.get()['status'], 200, 'init')
344
345        (_, sock) = self.http(
346            b"""GET / HTTP/1.1
347""",
348            start=True,
349            raw=True,
350            no_recv=True,
351        )
352
353        self.assertEqual(self.get()['status'], 200)
354
355        self.assertIn(
356            'success',
357            self.conf({"listeners": {}, "applications": {}}),
358            'reconfigure 3 clear configuration',
359        )
360
361        resp = self.http(
362            b"""Host: localhost
363Connection: close
364
365""",
366            sock=sock,
367            raw=True,
368        )
369
370        self.assertEqual(resp['status'], 200, 'reconfigure 3')
371
372    def test_python_atexit(self):
373        self.load('atexit')
374
375        self.get()
376
377        self.conf({"listeners": {}, "applications": {}})
378
379        self.stop()
380
381        self.assertIsNotNone(
382            self.wait_for_record(r'At exit called\.'), 'atexit'
383        )
384
385    @unittest.skip('not yet')
386    def test_python_application_start_response_exit(self):
387        self.load('start_response_exit')
388
389        self.assertEqual(self.get()['status'], 500, 'start response exit')
390
391    def test_python_application_input_iter(self):
392        self.load('input_iter')
393
394        body = '''0123456789
395next line
396
397last line'''
398
399        resp = self.post(body=body)
400        self.assertEqual(resp['body'], body, 'input iter')
401        self.assertEqual(
402            resp['headers']['X-Lines-Count'], '4', 'input iter lines'
403        )
404
405    def test_python_application_input_readline(self):
406        self.load('input_readline')
407
408        body = '''0123456789
409next line
410
411last line'''
412
413        resp = self.post(body=body)
414        self.assertEqual(resp['body'], body, 'input readline')
415        self.assertEqual(
416            resp['headers']['X-Lines-Count'], '4', 'input readline lines'
417        )
418
419    def test_python_application_input_readline_size(self):
420        self.load('input_readline_size')
421
422        body = '''0123456789
423next line
424
425last line'''
426
427        self.assertEqual(
428            self.post(body=body)['body'], body, 'input readline size'
429        )
430        self.assertEqual(
431            self.post(body='0123')['body'], '0123', 'input readline size less'
432        )
433
434    def test_python_application_input_readlines(self):
435        self.load('input_readlines')
436
437        body = '''0123456789
438next line
439
440last line'''
441
442        resp = self.post(body=body)
443        self.assertEqual(resp['body'], body, 'input readlines')
444        self.assertEqual(
445            resp['headers']['X-Lines-Count'], '4', 'input readlines lines'
446        )
447
448    def test_python_application_input_readlines_huge(self):
449        self.load('input_readlines')
450
451        body = (
452            '''0123456789 abcdefghi
453next line: 0123456789 abcdefghi
454
455last line: 987654321
456'''
457            * 512
458        )
459
460        self.assertEqual(
461            self.post(body=body, read_buffer_size=16384)['body'],
462            body,
463            'input readlines huge',
464        )
465
466    def test_python_application_input_read_length(self):
467        self.load('input_read_length')
468
469        body = '0123456789'
470
471        resp = self.post(
472            headers={
473                'Host': 'localhost',
474                'Input-Length': '5',
475                'Connection': 'close',
476            },
477            body=body,
478        )
479
480        self.assertEqual(resp['body'], body[:5], 'input read length lt body')
481
482        resp = self.post(
483            headers={
484                'Host': 'localhost',
485                'Input-Length': '15',
486                'Connection': 'close',
487            },
488            body=body,
489        )
490
491        self.assertEqual(resp['body'], body, 'input read length gt body')
492
493        resp = self.post(
494            headers={
495                'Host': 'localhost',
496                'Input-Length': '0',
497                'Connection': 'close',
498            },
499            body=body,
500        )
501
502        self.assertEqual(resp['body'], '', 'input read length zero')
503
504        resp = self.post(
505            headers={
506                'Host': 'localhost',
507                'Input-Length': '-1',
508                'Connection': 'close',
509            },
510            body=body,
511        )
512
513        self.assertEqual(resp['body'], body, 'input read length negative')
514
515    @unittest.skip('not yet')
516    def test_python_application_errors_write(self):
517        self.load('errors_write')
518
519        self.get()
520
521        self.stop()
522
523        self.assertIsNotNone(
524            self.wait_for_record(r'\[error\].+Error in application\.'),
525            'errors write',
526        )
527
528    def test_python_application_body_array(self):
529        self.load('body_array')
530
531        self.assertEqual(self.get()['body'], '0123456789', 'body array')
532
533    def test_python_application_body_io(self):
534        self.load('body_io')
535
536        self.assertEqual(self.get()['body'], '0123456789', 'body io')
537
538    def test_python_application_body_io_file(self):
539        self.load('body_io_file')
540
541        self.assertEqual(self.get()['body'], 'body\n', 'body io file')
542
543    @unittest.skip('not yet')
544    def test_python_application_syntax_error(self):
545        self.skip_alerts.append(r'Python failed to import module "wsgi"')
546        self.load('syntax_error')
547
548        self.assertEqual(self.get()['status'], 500, 'syntax error')
549
550    def test_python_application_close(self):
551        self.load('close')
552
553        self.get()
554
555        self.stop()
556
557        self.assertIsNotNone(self.wait_for_record(r'Close called\.'), 'close')
558
559    def test_python_application_close_error(self):
560        self.load('close_error')
561
562        self.get()
563
564        self.stop()
565
566        self.assertIsNotNone(
567            self.wait_for_record(r'Close called\.'), 'close error'
568        )
569
570    def test_python_application_not_iterable(self):
571        self.load('not_iterable')
572
573        self.get()
574
575        self.stop()
576
577        self.assertIsNotNone(
578            self.wait_for_record(
579                r'\[error\].+the application returned not an iterable object'
580            ),
581            'not iterable',
582        )
583
584    def test_python_application_write(self):
585        self.load('write')
586
587        self.assertEqual(self.get()['body'], '0123456789', 'write')
588
589    def test_python_application_threading(self):
590        """wait_for_record() timeouts after 5s while every thread works at
591        least 3s.  So without releasing GIL test should fail.
592        """
593
594        self.load('threading')
595
596        for _ in range(10):
597            self.get(no_recv=True)
598
599        self.assertIsNotNone(
600            self.wait_for_record(r'\(5\) Thread: 100'), 'last thread finished'
601        )
602
603    def test_python_application_iter_exception(self):
604        self.load('iter_exception')
605
606        # Default request doesn't lead to the exception.
607
608        resp = self.get(
609            headers={
610                'Host': 'localhost',
611                'X-Skip': '9',
612                'X-Chunked': '1',
613                'Connection': 'close',
614            }
615        )
616        self.assertEqual(resp['status'], 200, 'status')
617        self.assertEqual(resp['body'], 'XXXXXXX', 'body')
618
619        # Exception before start_response().
620
621        self.assertEqual(self.get()['status'], 503, 'error')
622
623        self.assertIsNotNone(self.wait_for_record(r'Traceback'), 'traceback')
624        self.assertIsNotNone(
625            self.wait_for_record(r'raise Exception\(\'first exception\'\)'),
626            'first exception raise',
627        )
628        self.assertEqual(
629            len(self.findall(r'Traceback')), 1, 'traceback count 1'
630        )
631
632        # Exception after start_response(), before first write().
633
634        self.assertEqual(
635            self.get(
636                headers={
637                    'Host': 'localhost',
638                    'X-Skip': '1',
639                    'Connection': 'close',
640                }
641            )['status'],
642            503,
643            'error 2',
644        )
645
646        self.assertIsNotNone(
647            self.wait_for_record(r'raise Exception\(\'second exception\'\)'),
648            'exception raise second',
649        )
650        self.assertEqual(
651            len(self.findall(r'Traceback')), 2, 'traceback count 2'
652        )
653
654        # Exception after first write(), before first __next__().
655
656        _, sock = self.get(
657            headers={
658                'Host': 'localhost',
659                'X-Skip': '2',
660                'Connection': 'keep-alive',
661            },
662            start=True,
663        )
664
665        self.assertIsNotNone(
666            self.wait_for_record(r'raise Exception\(\'third exception\'\)'),
667            'exception raise third',
668        )
669        self.assertEqual(
670            len(self.findall(r'Traceback')), 3, 'traceback count 3'
671        )
672
673        self.assertDictEqual(self.get(sock=sock), {}, 'closed connection')
674
675        # Exception after first write(), before first __next__(),
676        # chunked (incomplete body).
677
678        resp = self.get(
679            headers={
680                'Host': 'localhost',
681                'X-Skip': '2',
682                'X-Chunked': '1',
683                'Connection': 'close',
684            },
685            raw_resp=True
686        )
687        if resp:
688            self.assertNotEqual(resp[-5:], '0\r\n\r\n', 'incomplete body')
689        self.assertEqual(
690            len(self.findall(r'Traceback')), 4, 'traceback count 4'
691        )
692
693        # Exception in __next__().
694
695        _, sock = self.get(
696            headers={
697                'Host': 'localhost',
698                'X-Skip': '3',
699                'Connection': 'keep-alive',
700            },
701            start=True,
702        )
703
704        self.assertIsNotNone(
705            self.wait_for_record(r'raise Exception\(\'next exception\'\)'),
706            'exception raise next',
707        )
708        self.assertEqual(
709            len(self.findall(r'Traceback')), 5, 'traceback count 5'
710        )
711
712        self.assertDictEqual(self.get(sock=sock), {}, 'closed connection 2')
713
714        # Exception in __next__(), chunked (incomplete body).
715
716        resp = self.get(
717            headers={
718                'Host': 'localhost',
719                'X-Skip': '3',
720                'X-Chunked': '1',
721                'Connection': 'close',
722            },
723            raw_resp=True
724        )
725        if resp:
726            self.assertNotEqual(resp[-5:], '0\r\n\r\n', 'incomplete body 2')
727        self.assertEqual(
728            len(self.findall(r'Traceback')), 6, 'traceback count 6'
729        )
730
731        # Exception before start_response() and in close().
732
733        self.assertEqual(
734            self.get(
735                headers={
736                    'Host': 'localhost',
737                    'X-Not-Skip-Close': '1',
738                    'Connection': 'close',
739                }
740            )['status'],
741            503,
742            'error',
743        )
744
745        self.assertIsNotNone(
746            self.wait_for_record(r'raise Exception\(\'close exception\'\)'),
747            'exception raise close',
748        )
749        self.assertEqual(
750            len(self.findall(r'Traceback')), 8, 'traceback count 8'
751        )
752
753    def test_python_user_group(self):
754        if not self.is_su:
755            print("requires root")
756            raise unittest.SkipTest()
757
758        nobody_uid = pwd.getpwnam('nobody').pw_uid
759
760        group = 'nobody'
761
762        try:
763            group_id = grp.getgrnam(group).gr_gid
764        except:
765            group = 'nogroup'
766            group_id = grp.getgrnam(group).gr_gid
767
768        self.load('user_group')
769
770        obj = self.getjson()['body']
771        self.assertEqual(obj['UID'], nobody_uid, 'nobody uid')
772        self.assertEqual(obj['GID'], group_id, 'nobody gid')
773
774        self.load('user_group', user='nobody')
775
776        obj = self.getjson()['body']
777        self.assertEqual(obj['UID'], nobody_uid, 'nobody uid user=nobody')
778        self.assertEqual(obj['GID'], group_id, 'nobody gid user=nobody')
779
780        self.load('user_group', user='nobody', group=group)
781
782        obj = self.getjson()['body']
783        self.assertEqual(
784            obj['UID'], nobody_uid, 'nobody uid user=nobody group=%s' % group
785        )
786
787        self.assertEqual(
788            obj['GID'], group_id, 'nobody gid user=nobody group=%s' % group
789        )
790
791        self.load('user_group', group=group)
792
793        obj = self.getjson()['body']
794        self.assertEqual(
795            obj['UID'], nobody_uid, 'nobody uid group=%s' % group
796        )
797
798        self.assertEqual(obj['GID'], group_id, 'nobody gid group=%s' % group)
799
800        self.load('user_group', user='root')
801
802        obj = self.getjson()['body']
803        self.assertEqual(obj['UID'], 0, 'root uid user=root')
804        self.assertEqual(obj['GID'], 0, 'root gid user=root')
805
806        group = 'root'
807
808        try:
809            grp.getgrnam(group)
810            group = True
811        except:
812            group = False
813
814        if group:
815            self.load('user_group', user='root', group='root')
816
817            obj = self.getjson()['body']
818            self.assertEqual(obj['UID'], 0, 'root uid user=root group=root')
819            self.assertEqual(obj['GID'], 0, 'root gid user=root group=root')
820
821            self.load('user_group', group='root')
822
823            obj = self.getjson()['body']
824            self.assertEqual(obj['UID'], nobody_uid, 'root uid group=root')
825            self.assertEqual(obj['GID'], 0, 'root gid group=root')
826
827if __name__ == '__main__':
828    TestPythonApplication.main()
829