xref: /unit/src/nxt_openssl.c (revision 2617:18a10bb7346d)
1 
2 /*
3  * Copyright (C) Igor Sysoev
4  * Copyright (C) NGINX, Inc.
5  */
6 
7 #include <nxt_main.h>
8 #include <nxt_conf.h>
9 
10 #define OPENSSL_SUPPRESS_DEPRECATED
11 
12 #include <openssl/ssl.h>
13 #include <openssl/conf.h>
14 #include <openssl/err.h>
15 #include <openssl/rand.h>
16 #include <openssl/x509v3.h>
17 #include <openssl/bio.h>
18 #include <openssl/evp.h>
19 
20 
21 typedef struct {
22     SSL               *session;
23     nxt_conn_t        *conn;
24 
25     int               ssl_error;
26     uint8_t           times;      /* 2 bits */
27     uint8_t           handshake;  /* 1 bit  */
28 
29     nxt_tls_conf_t    *conf;
30     nxt_buf_mem_t     buffer;
31 } nxt_openssl_conn_t;
32 
33 
34 struct nxt_tls_ticket_s {
35     u_char            name[16];
36     u_char            hmac_key[32];
37     u_char            aes_key[32];
38     uint8_t           size;
39 };
40 
41 
42 struct nxt_tls_tickets_s {
43     nxt_uint_t        count;
44     nxt_tls_ticket_t  tickets[];
45 };
46 
47 
48 typedef enum {
49     NXT_OPENSSL_HANDSHAKE = 0,
50     NXT_OPENSSL_READ,
51     NXT_OPENSSL_WRITE,
52     NXT_OPENSSL_SHUTDOWN,
53 } nxt_openssl_io_t;
54 
55 
56 static nxt_int_t nxt_openssl_library_init(nxt_task_t *task);
57 static void nxt_openssl_library_free(nxt_task_t *task);
58 #if OPENSSL_VERSION_NUMBER < 0x10100004L
59 static nxt_int_t nxt_openssl_locks_init(void);
60 static void nxt_openssl_lock(int mode, int type, const char *file, int line);
61 static unsigned long nxt_openssl_thread_id(void);
62 static void nxt_openssl_locks_free(void);
63 #endif
64 static nxt_int_t nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp,
65     nxt_tls_init_t *tls_init, nxt_bool_t last);
66 static nxt_int_t nxt_openssl_chain_file(nxt_task_t *task, SSL_CTX *ctx,
67     nxt_tls_conf_t *conf, nxt_mp_t *mp, nxt_bool_t single);
68 #if (NXT_HAVE_OPENSSL_CONF_CMD)
69 static nxt_int_t nxt_ssl_conf_commands(nxt_task_t *task, SSL_CTX *ctx,
70     nxt_conf_value_t *value, nxt_mp_t *mp);
71 #endif
72 #if (NXT_HAVE_OPENSSL_TLSEXT)
73 static nxt_int_t nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx,
74     nxt_tls_init_t *tls_init, nxt_mp_t *mp);
75 static int nxt_tls_ticket_key_callback(SSL *s, unsigned char *name,
76     unsigned char *iv, EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc);
77 #endif
78 static void nxt_ssl_session_cache(SSL_CTX *ctx, size_t cache_size,
79     time_t timeout);
80 static nxt_uint_t nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert,
81     nxt_tls_conf_t *conf, nxt_mp_t *mp);
82 static nxt_int_t nxt_openssl_bundle_hash_test(nxt_lvlhsh_query_t *lhq,
83     void *data);
84 static nxt_int_t nxt_openssl_bundle_hash_insert(nxt_task_t *task,
85     nxt_lvlhsh_t *lvlhsh, nxt_tls_bundle_hash_item_t *item, nxt_mp_t * mp);
86 static nxt_int_t nxt_openssl_servername(SSL *s, int *ad, void *arg);
87 static nxt_tls_bundle_conf_t *nxt_openssl_find_ctx(nxt_tls_conf_t *conf,
88     nxt_str_t *sn);
89 static void nxt_openssl_server_free(nxt_task_t *task, nxt_tls_conf_t *conf);
90 static void nxt_openssl_conn_init(nxt_task_t *task, nxt_tls_conf_t *conf,
91     nxt_conn_t *c);
92 static void nxt_openssl_conn_handshake(nxt_task_t *task, void *obj, void *data);
93 static ssize_t nxt_openssl_conn_io_recvbuf(nxt_conn_t *c, nxt_buf_t *b);
94 static ssize_t nxt_openssl_conn_io_sendbuf(nxt_task_t *task, nxt_sendbuf_t *sb);
95 static ssize_t nxt_openssl_conn_io_send(nxt_task_t *task, nxt_sendbuf_t *sb,
96     void *buf, size_t size);
97 static void nxt_openssl_conn_io_shutdown(nxt_task_t *task, void *obj,
98     void *data);
99 static nxt_int_t nxt_openssl_conn_test_error(nxt_task_t *task, nxt_conn_t *c,
100     int ret, nxt_err_t sys_err, nxt_openssl_io_t io);
101 static void nxt_openssl_conn_io_shutdown_timeout(nxt_task_t *task, void *obj,
102     void *data);
103 static void nxt_cdecl nxt_openssl_conn_error(nxt_task_t *task,
104     nxt_err_t err, const char *fmt, ...);
105 static nxt_uint_t nxt_openssl_log_error_level(nxt_err_t err);
106 
107 
108 const nxt_tls_lib_t  nxt_openssl_lib = {
109     .library_init = nxt_openssl_library_init,
110     .library_free = nxt_openssl_library_free,
111 
112     .server_init = nxt_openssl_server_init,
113     .server_free = nxt_openssl_server_free,
114 };
115 
116 
117 static nxt_conn_io_t  nxt_openssl_conn_io = {
118     .read = nxt_conn_io_read,
119     .recvbuf = nxt_openssl_conn_io_recvbuf,
120 
121     .write = nxt_conn_io_write,
122     .sendbuf = nxt_openssl_conn_io_sendbuf,
123 
124     .shutdown = nxt_openssl_conn_io_shutdown,
125 };
126 
127 
128 static long  nxt_openssl_version;
129 static int   nxt_openssl_connection_index;
130 
131 
132 static nxt_int_t
nxt_openssl_library_init(nxt_task_t * task)133 nxt_openssl_library_init(nxt_task_t *task)
134 {
135     int  index;
136 
137     if (nxt_fast_path(nxt_openssl_version != 0)) {
138         return NXT_OK;
139     }
140 
141 #if OPENSSL_VERSION_NUMBER >= 0x10100003L
142 
143     OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL);
144 
145 #else
146     {
147         nxt_int_t  ret;
148 
149         SSL_load_error_strings();
150 
151         OPENSSL_config(NULL);
152 
153         /*
154          * SSL_library_init(3):
155          *
156          *   SSL_library_init() always returns "1",
157          *   so it is safe to discard the return value.
158          */
159         (void) SSL_library_init();
160 
161         ret = nxt_openssl_locks_init();
162         if (nxt_slow_path(ret != NXT_OK)) {
163             return ret;
164         }
165     }
166 
167 #endif
168 
169     nxt_openssl_version = SSLeay();
170 
171     nxt_log(task, NXT_LOG_INFO, "%s, %xl",
172             SSLeay_version(SSLEAY_VERSION), nxt_openssl_version);
173 
174 #ifndef SSL_OP_NO_COMPRESSION
175     {
176         /*
177          * Disable gzip compression in OpenSSL prior to 1.0.0
178          * version, this saves about 522K per connection.
179          */
180         int                 n;
181         STACK_OF(SSL_COMP)  *ssl_comp_methods;
182 
183         ssl_comp_methods = SSL_COMP_get_compression_methods();
184 
185         for (n = sk_SSL_COMP_num(ssl_comp_methods); n != 0; n--) {
186             (void) sk_SSL_COMP_pop(ssl_comp_methods);
187         }
188     }
189 #endif
190 
191     index = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL);
192 
193     if (index == -1) {
194         nxt_openssl_log_error(task, NXT_LOG_ALERT,
195                               "SSL_get_ex_new_index() failed");
196         return NXT_ERROR;
197     }
198 
199     nxt_openssl_connection_index = index;
200 
201     return NXT_OK;
202 }
203 
204 
205 #if OPENSSL_VERSION_NUMBER >= 0x10100003L
206 
207 static void
nxt_openssl_library_free(nxt_task_t * task)208 nxt_openssl_library_free(nxt_task_t *task)
209 {
210 }
211 
212 #else
213 
214 static nxt_thread_mutex_t  *nxt_openssl_locks;
215 
216 static nxt_int_t
nxt_openssl_locks_init(void)217 nxt_openssl_locks_init(void)
218 {
219     int        i, n;
220     nxt_int_t  ret;
221 
222     n = CRYPTO_num_locks();
223 
224     nxt_openssl_locks = OPENSSL_malloc(n * sizeof(nxt_thread_mutex_t));
225     if (nxt_slow_path(nxt_openssl_locks == NULL)) {
226         return NXT_ERROR;
227     }
228 
229     for (i = 0; i < n; i++) {
230         ret = nxt_thread_mutex_create(&nxt_openssl_locks[i]);
231         if (nxt_slow_path(ret != NXT_OK)) {
232             return ret;
233         }
234     }
235 
236     CRYPTO_set_locking_callback(nxt_openssl_lock);
237 
238     CRYPTO_set_id_callback(nxt_openssl_thread_id);
239 
240     return NXT_OK;
241 }
242 
243 
244 static void
nxt_openssl_lock(int mode,int type,const char * file,int line)245 nxt_openssl_lock(int mode, int type, const char *file, int line)
246 {
247     nxt_thread_mutex_t  *lock;
248 
249     lock = &nxt_openssl_locks[type];
250 
251     if ((mode & CRYPTO_LOCK) != 0) {
252         (void) nxt_thread_mutex_lock(lock);
253 
254     } else {
255         (void) nxt_thread_mutex_unlock(lock);
256     }
257 }
258 
259 
260 static u_long
nxt_openssl_thread_id(void)261 nxt_openssl_thread_id(void)
262 {
263     return (u_long) nxt_thread_handle();
264 }
265 
266 
267 static void
nxt_openssl_library_free(nxt_task_t * task)268 nxt_openssl_library_free(nxt_task_t *task)
269 {
270     nxt_openssl_locks_free();
271 }
272 
273 
274 static void
nxt_openssl_locks_free(void)275 nxt_openssl_locks_free(void)
276 {
277     int  i, n;
278 
279     n = CRYPTO_num_locks();
280 
281     CRYPTO_set_locking_callback(NULL);
282 
283     for (i = 0; i < n; i++) {
284         nxt_thread_mutex_destroy(&nxt_openssl_locks[i]);
285     }
286 
287     OPENSSL_free(nxt_openssl_locks);
288 }
289 
290 #endif
291 
292 
293 static nxt_int_t
nxt_openssl_server_init(nxt_task_t * task,nxt_mp_t * mp,nxt_tls_init_t * tls_init,nxt_bool_t last)294 nxt_openssl_server_init(nxt_task_t *task, nxt_mp_t *mp,
295     nxt_tls_init_t *tls_init, nxt_bool_t last)
296 {
297     SSL_CTX                *ctx;
298     const char             *ca_certificate;
299     nxt_tls_conf_t         *conf;
300     STACK_OF(X509_NAME)    *list;
301     nxt_tls_bundle_conf_t  *bundle;
302 
303     ctx = SSL_CTX_new(SSLv23_server_method());
304     if (ctx == NULL) {
305         nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_CTX_new() failed");
306         return NXT_ERROR;
307     }
308 
309     conf = tls_init->conf;
310 
311     bundle = conf->bundle;
312     nxt_assert(bundle != NULL);
313 
314     bundle->ctx = ctx;
315 
316 #ifdef SSL_OP_NO_RENEGOTIATION
317     /* Renegration is not currently supported. */
318     SSL_CTX_set_options(ctx, SSL_OP_NO_RENEGOTIATION);
319 #endif
320 
321 #ifdef SSL_OP_NO_COMPRESSION
322     /*
323      * Disable gzip compression in OpenSSL 1.0.0,
324      * this saves about 522K per connection.
325      */
326     SSL_CTX_set_options(ctx, SSL_OP_NO_COMPRESSION);
327 #endif
328 
329 #ifdef SSL_OP_IGNORE_UNEXPECTED_EOF
330     /* Request SSL_ERROR_ZERO_RETURN on EOF. */
331     SSL_CTX_set_options(ctx, SSL_OP_IGNORE_UNEXPECTED_EOF);
332 #endif
333 
334 #ifdef SSL_MODE_RELEASE_BUFFERS
335 
336     if (nxt_openssl_version >= 10001078) {
337         /*
338          * Allow to release read and write buffers in OpenSSL 1.0.0,
339          * this saves about 34K per idle connection.  It is not safe
340          * before OpenSSL 1.0.1h (CVE-2010-5298).
341          */
342         SSL_CTX_set_mode(ctx, SSL_MODE_RELEASE_BUFFERS);
343     }
344 
345 #endif
346 
347     if (nxt_openssl_chain_file(task, ctx, conf, mp,
348                                last && bundle->next == NULL)
349         != NXT_OK)
350     {
351         goto fail;
352     }
353 /*
354     key = conf->certificate_key;
355 
356     if (SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) == 0) {
357         nxt_openssl_log_error(task, NXT_LOG_ALERT,
358                               "SSL_CTX_use_PrivateKey_file(\"%s\") failed",
359                               key);
360         goto fail;
361     }
362 */
363 
364     if (conf->ciphers) {  /* else use system crypto policy */
365         if (SSL_CTX_set_cipher_list(ctx, conf->ciphers) == 0) {
366             nxt_openssl_log_error(task, NXT_LOG_ALERT,
367                               "SSL_CTX_set_cipher_list(\"%s\") failed",
368                               conf->ciphers);
369             goto fail;
370         }
371     }
372 
373 #if (NXT_HAVE_OPENSSL_CONF_CMD)
374     if (tls_init->conf_cmds != NULL
375         && nxt_ssl_conf_commands(task, ctx, tls_init->conf_cmds, mp) != NXT_OK)
376     {
377         goto fail;
378     }
379 #endif
380 
381     nxt_ssl_session_cache(ctx, tls_init->cache_size, tls_init->timeout);
382 
383 #if (NXT_HAVE_OPENSSL_TLSEXT)
384     if (nxt_tls_ticket_keys(task, ctx, tls_init, mp) != NXT_OK) {
385         goto fail;
386     }
387 #endif
388 
389     SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE);
390 
391     if (conf->ca_certificate != NULL) {
392 
393         /* TODO: verify callback */
394         SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL);
395 
396         /* TODO: verify depth */
397         SSL_CTX_set_verify_depth(ctx, 1);
398 
399         ca_certificate = conf->ca_certificate;
400 
401         if (SSL_CTX_load_verify_locations(ctx, ca_certificate, NULL) == 0) {
402             nxt_openssl_log_error(task, NXT_LOG_ALERT,
403                               "SSL_CTX_load_verify_locations(\"%s\") failed",
404                               ca_certificate);
405             goto fail;
406         }
407 
408         list = SSL_load_client_CA_file(ca_certificate);
409 
410         if (list == NULL) {
411             nxt_openssl_log_error(task, NXT_LOG_ALERT,
412                               "SSL_load_client_CA_file(\"%s\") failed",
413                               ca_certificate);
414             goto fail;
415         }
416 
417         /*
418          * SSL_load_client_CA_file() in OpenSSL prior to 0.9.7h and
419          * 0.9.8 versions always leaves an error in the error queue.
420          */
421         ERR_clear_error();
422 
423         SSL_CTX_set_client_CA_list(ctx, list);
424     }
425 
426     if (last) {
427         conf->conn_init = nxt_openssl_conn_init;
428 
429         if (bundle->next != NULL) {
430             SSL_CTX_set_tlsext_servername_callback(ctx, nxt_openssl_servername);
431         }
432     }
433 
434     return NXT_OK;
435 
436 fail:
437 
438     SSL_CTX_free(ctx);
439 
440 #if (OPENSSL_VERSION_NUMBER >= 0x1010100fL \
441      && OPENSSL_VERSION_NUMBER < 0x1010101fL)
442     RAND_keep_random_devices_open(0);
443 #endif
444 
445     return NXT_ERROR;
446 }
447 
448 
449 static nxt_int_t
nxt_openssl_chain_file(nxt_task_t * task,SSL_CTX * ctx,nxt_tls_conf_t * conf,nxt_mp_t * mp,nxt_bool_t single)450 nxt_openssl_chain_file(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_conf_t *conf,
451     nxt_mp_t *mp, nxt_bool_t single)
452 {
453     BIO                    *bio;
454     X509                   *cert, *ca;
455     long                   reason;
456     EVP_PKEY               *key;
457     nxt_int_t              ret;
458     nxt_tls_bundle_conf_t  *bundle;
459 
460     ret = NXT_ERROR;
461     cert = NULL;
462 
463     bio = BIO_new(BIO_s_fd());
464     if (bio == NULL) {
465         goto end;
466     }
467 
468     bundle = conf->bundle;
469 
470     BIO_set_fd(bio, bundle->chain_file, BIO_CLOSE);
471 
472     cert = PEM_read_bio_X509_AUX(bio, NULL, NULL, NULL);
473     if (cert == NULL) {
474         goto end;
475     }
476 
477     if (SSL_CTX_use_certificate(ctx, cert) != 1) {
478         goto end;
479     }
480 
481     if (!single && nxt_openssl_cert_get_names(task, cert, conf, mp) != NXT_OK) {
482         goto clean;
483     }
484 
485     for ( ;; ) {
486         ca = PEM_read_bio_X509(bio, NULL, NULL, NULL);
487 
488         if (ca == NULL) {
489             reason = ERR_GET_REASON(ERR_peek_last_error());
490             if (reason != PEM_R_NO_START_LINE) {
491                 goto end;
492             }
493 
494             ERR_clear_error();
495             break;
496         }
497 
498         /*
499          * Note that ca isn't freed if it was successfully added to the chain,
500          * while the main certificate needs a X509_free() call, since
501          * its reference count is increased by SSL_CTX_use_certificate().
502          */
503 #ifdef SSL_CTX_add0_chain_cert
504         if (SSL_CTX_add0_chain_cert(ctx, ca) != 1) {
505 #else
506         if (SSL_CTX_add_extra_chain_cert(ctx, ca) != 1) {
507 #endif
508             X509_free(ca);
509             goto end;
510         }
511     }
512 
513     if (BIO_reset(bio) != 0) {
514         goto end;
515     }
516 
517     key = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL);
518     if (key == NULL) {
519         goto end;
520     }
521 
522     if (SSL_CTX_use_PrivateKey(ctx, key) == 1) {
523         ret = NXT_OK;
524     }
525 
526     EVP_PKEY_free(key);
527 
528 end:
529 
530     if (ret != NXT_OK) {
531         nxt_openssl_log_error(task, NXT_LOG_ALERT,
532                               "nxt_openssl_chain_file() failed");
533     }
534 
535 clean:
536 
537     BIO_free(bio);
538     X509_free(cert);
539 
540     return ret;
541 }
542 
543 
544 #if (NXT_HAVE_OPENSSL_CONF_CMD)
545 
546 static nxt_int_t
547 nxt_ssl_conf_commands(nxt_task_t *task, SSL_CTX *ctx, nxt_conf_value_t *conf,
548     nxt_mp_t *mp)
549 {
550     int               ret;
551     char              *zcmd, *zvalue;
552     uint32_t          index;
553     nxt_str_t         cmd, value;
554     SSL_CONF_CTX      *cctx;
555     nxt_conf_value_t  *member;
556 
557     cctx = SSL_CONF_CTX_new();
558     if (nxt_slow_path(cctx == NULL)) {
559         nxt_openssl_log_error(task, NXT_LOG_ALERT,
560                               "SSL_CONF_CTX_new() failed");
561         return NXT_ERROR;
562     }
563 
564     SSL_CONF_CTX_set_flags(cctx, SSL_CONF_FLAG_FILE
565                                  | SSL_CONF_FLAG_SERVER
566                                  | SSL_CONF_FLAG_CERTIFICATE
567                                  | SSL_CONF_FLAG_SHOW_ERRORS);
568 
569     SSL_CONF_CTX_set_ssl_ctx(cctx, ctx);
570 
571     index = 0;
572 
573     for ( ;; ) {
574         member = nxt_conf_next_object_member(conf, &cmd, &index);
575         if (nxt_slow_path(member == NULL)) {
576             break;
577         }
578 
579         nxt_conf_get_string(member, &value);
580 
581         zcmd = nxt_str_cstrz(mp, &cmd);
582         zvalue = nxt_str_cstrz(mp, &value);
583 
584         if (nxt_slow_path(zcmd == NULL || zvalue == NULL)) {
585             goto fail;
586         }
587 
588         ret = SSL_CONF_cmd(cctx, zcmd, zvalue);
589         if (ret == -2) {
590             nxt_openssl_log_error(task, NXT_LOG_ERR,
591                                   "unknown command \"%s\" in "
592                                   "\"conf_commands\" option", zcmd);
593             goto fail;
594         }
595 
596         if (ret <= 0) {
597             nxt_openssl_log_error(task, NXT_LOG_ERR,
598                                   "invalid value \"%s\" for command \"%s\" "
599                                   "in \"conf_commands\" option",
600                                   zvalue, zcmd);
601             goto fail;
602         }
603 
604         nxt_debug(task, "SSL_CONF_cmd(\"%s\", \"%s\")", zcmd, zvalue);
605     }
606 
607     if (SSL_CONF_CTX_finish(cctx) != 1) {
608         nxt_openssl_log_error(task, NXT_LOG_ALERT,
609                               "SSL_CONF_finish() failed");
610         goto fail;
611     }
612 
613     SSL_CONF_CTX_free(cctx);
614 
615     return NXT_OK;
616 
617 fail:
618 
619     SSL_CONF_CTX_free(cctx);
620 
621     return NXT_ERROR;
622 }
623 
624 #endif
625 
626 #if (NXT_HAVE_OPENSSL_TLSEXT)
627 
628 static nxt_int_t
629 nxt_tls_ticket_keys(nxt_task_t *task, SSL_CTX *ctx, nxt_tls_init_t *tls_init,
630     nxt_mp_t *mp)
631 {
632     size_t             len;
633     uint32_t           i;
634     nxt_str_t          value;
635     nxt_uint_t         count;
636     nxt_conf_value_t   *member, *tickets_conf;
637     nxt_tls_ticket_t   *ticket;
638     nxt_tls_tickets_t  *tickets;
639     u_char             buf[80];
640 
641     tickets_conf = tls_init->tickets_conf;
642 
643     if (tickets_conf == NULL) {
644         goto no_ticket;
645     }
646 
647     if (nxt_conf_type(tickets_conf) == NXT_CONF_BOOLEAN) {
648         if (nxt_conf_get_boolean(tickets_conf) == 0) {
649             goto no_ticket;
650         }
651 
652         return NXT_OK;
653     }
654 
655     count = nxt_conf_array_elements_count_or_1(tickets_conf);
656 
657     if (count == 0) {
658         goto no_ticket;
659     }
660 
661 #ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB
662 
663     tickets = nxt_mp_get(mp, sizeof(nxt_tls_tickets_t)
664                              + count * sizeof(nxt_tls_ticket_t));
665     if (nxt_slow_path(tickets == NULL)) {
666         return NXT_ERROR;
667     }
668 
669     tickets->count = count;
670     tls_init->conf->tickets = tickets;
671     i = 0;
672 
673     do {
674         ticket = &tickets->tickets[i];
675 
676         i++;
677 
678         member = nxt_conf_get_array_element_or_itself(tickets_conf, count - i);
679         if (member == NULL) {
680             break;
681         }
682 
683         nxt_conf_get_string(member, &value);
684 
685         len = nxt_base64_decode(buf, value.start, value.length);
686 
687         nxt_memcpy(ticket->name, buf, 16);
688 
689         if (len == 48) {
690             nxt_memcpy(ticket->aes_key, buf + 16, 16);
691             nxt_memcpy(ticket->hmac_key, buf + 32, 16);
692             ticket->size = 16;
693 
694         } else {
695             nxt_memcpy(ticket->hmac_key, buf + 16, 32);
696             nxt_memcpy(ticket->aes_key, buf + 48, 32);
697             ticket->size = 32;
698         }
699 
700     } while (i < count);
701 
702     if (SSL_CTX_set_tlsext_ticket_key_cb(ctx, nxt_tls_ticket_key_callback)
703         == 0)
704     {
705         nxt_openssl_log_error(task, NXT_LOG_ALERT,
706                       "Unit was built with Session Tickets support, however, "
707                       "now it is linked dynamically to an OpenSSL library "
708                       "which has no tlsext support, therefore Session Tickets "
709                       "are not available");
710 
711         return NXT_ERROR;
712     }
713 
714     return NXT_OK;
715 
716 #else
717     nxt_alert(task, "Setting custom session ticket keys is not supported with "
718                     "this version of OpenSSL library");
719 
720     return NXT_ERROR;
721 
722 #endif
723 
724 no_ticket:
725 
726     SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET);
727 
728     return NXT_OK;
729 }
730 
731 
732 #ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB
733 
734 static int
735 nxt_tls_ticket_key_callback(SSL *s, unsigned char *name, unsigned char *iv,
736     EVP_CIPHER_CTX *ectx, HMAC_CTX *hctx, int enc)
737 {
738     nxt_uint_t          i;
739     nxt_conn_t          *c;
740     const EVP_MD        *digest;
741     const EVP_CIPHER    *cipher;
742     nxt_tls_ticket_t    *ticket;
743     nxt_openssl_conn_t  *tls;
744 
745     c = SSL_get_ex_data(s, nxt_openssl_connection_index);
746 
747     if (nxt_slow_path(c == NULL)) {
748         nxt_thread_log_alert("SSL_get_ex_data() failed");
749         return -1;
750     }
751 
752     tls = c->u.tls;
753     ticket = tls->conf->tickets->tickets;
754 
755     i = 0;
756 
757     if (enc == 1) {
758         /* encrypt session ticket */
759 
760         nxt_debug(c->socket.task, "TLS session ticket encrypt");
761 
762         cipher = (ticket[0].size == 16) ? EVP_aes_128_cbc() : EVP_aes_256_cbc();
763 
764         if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1) {
765             nxt_openssl_log_error(c->socket.task, NXT_LOG_ALERT,
766                                   "RAND_bytes() failed");
767             return -1;
768         }
769 
770         nxt_memcpy(name, ticket[0].name, 16);
771 
772         if (EVP_EncryptInit_ex(ectx, cipher, NULL, ticket[0].aes_key, iv) != 1)
773         {
774             nxt_openssl_log_error(c->socket.task, NXT_LOG_ALERT,
775                                   "EVP_EncryptInit_ex() failed");
776             return -1;
777         }
778 
779     } else {
780         /* decrypt session ticket */
781 
782         do {
783             if (memcmp(name, ticket[i].name, 16) == 0) {
784                 goto found;
785             }
786 
787         } while (++i < tls->conf->tickets->count);
788 
789         nxt_debug(c->socket.task, "TLS session ticket decrypt, key not found");
790 
791         return 0;
792 
793     found:
794 
795         nxt_debug(c->socket.task,
796                   "TLS session ticket decrypt, key number: \"%d\"", i);
797 
798         enc = (i == 0) ? 1 : 2 /* renew */;
799 
800         cipher = (ticket[i].size == 16) ? EVP_aes_128_cbc() : EVP_aes_256_cbc();
801 
802         if (EVP_DecryptInit_ex(ectx, cipher, NULL, ticket[i].aes_key, iv) != 1)
803         {
804             nxt_openssl_log_error(c->socket.task, NXT_LOG_ALERT,
805                                   "EVP_DecryptInit_ex() failed");
806             return -1;
807         }
808     }
809 
810 #ifdef OPENSSL_NO_SHA256
811     digest = EVP_sha1();
812 #else
813     digest = EVP_sha256();
814 #endif
815 
816     if (HMAC_Init_ex(hctx, ticket[i].hmac_key, ticket[i].size, digest, NULL)
817         != 1)
818     {
819         nxt_openssl_log_error(c->socket.task, NXT_LOG_ALERT,
820                               "HMAC_Init_ex() failed");
821         return -1;
822     }
823 
824     return enc;
825 }
826 
827 #endif /* SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB */
828 
829 #endif /* NXT_HAVE_OPENSSL_TLSEXT */
830 
831 
832 static void
833 nxt_ssl_session_cache(SSL_CTX *ctx, size_t cache_size, time_t timeout)
834 {
835     if (cache_size == 0) {
836         SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF);
837         return;
838     }
839 
840     SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER);
841 
842     SSL_CTX_sess_set_cache_size(ctx, cache_size);
843 
844     SSL_CTX_set_timeout(ctx, (long) timeout);
845 }
846 
847 
848 static nxt_uint_t
849 nxt_openssl_cert_get_names(nxt_task_t *task, X509 *cert, nxt_tls_conf_t *conf,
850     nxt_mp_t *mp)
851 {
852     int                         len;
853     nxt_str_t                   domain, str;
854     X509_NAME                   *x509_name;
855     nxt_uint_t                  i, n;
856     GENERAL_NAME                *name;
857     nxt_tls_bundle_conf_t       *bundle;
858     STACK_OF(GENERAL_NAME)      *alt_names;
859     nxt_tls_bundle_hash_item_t  *item;
860 
861     bundle = conf->bundle;
862 
863     alt_names = X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL);
864 
865     if (alt_names != NULL) {
866         n = sk_GENERAL_NAME_num(alt_names);
867 
868         for (i = 0; i != n; i++) {
869             name = sk_GENERAL_NAME_value(alt_names, i);
870 
871             if (name->type != GEN_DNS) {
872                 continue;
873             }
874 
875             str.length = ASN1_STRING_length(name->d.dNSName);
876 #if OPENSSL_VERSION_NUMBER > 0x10100000L
877             str.start = (u_char *) ASN1_STRING_get0_data(name->d.dNSName);
878 #else
879             str.start = ASN1_STRING_data(name->d.dNSName);
880 #endif
881 
882             domain.start = nxt_mp_nget(mp, str.length);
883             if (nxt_slow_path(domain.start == NULL)) {
884                 goto fail;
885             }
886 
887             domain.length = str.length;
888             nxt_memcpy_lowcase(domain.start, str.start, str.length);
889 
890             item = nxt_mp_get(mp, sizeof(nxt_tls_bundle_hash_item_t));
891             if (nxt_slow_path(item == NULL)) {
892                 goto fail;
893             }
894 
895             item->name = domain;
896             item->bundle = bundle;
897 
898             if (nxt_openssl_bundle_hash_insert(task, &conf->bundle_hash,
899                                                item, mp)
900                 == NXT_ERROR)
901             {
902                 goto fail;
903             }
904         }
905 
906         sk_GENERAL_NAME_pop_free(alt_names, GENERAL_NAME_free);
907 
908     } else {
909         x509_name = X509_get_subject_name(cert);
910         len = X509_NAME_get_text_by_NID(x509_name, NID_commonName,
911                                         NULL, 0);
912         if (len <= 0) {
913             nxt_log(task, NXT_LOG_WARN, "certificate \"%V\" has neither "
914                     "Subject Alternative Name nor Common Name", &bundle->name);
915             return NXT_OK;
916         }
917 
918         domain.start = nxt_mp_nget(mp, len + 1);
919         if (nxt_slow_path(domain.start == NULL)) {
920             return NXT_ERROR;
921         }
922 
923         domain.length = X509_NAME_get_text_by_NID(x509_name, NID_commonName,
924                                                   (char *) domain.start,
925                                                   len + 1);
926         nxt_memcpy_lowcase(domain.start, domain.start, domain.length);
927 
928         item = nxt_mp_get(mp, sizeof(nxt_tls_bundle_hash_item_t));
929         if (nxt_slow_path(item == NULL)) {
930             return NXT_ERROR;
931         }
932 
933         item->name = domain;
934         item->bundle = bundle;
935 
936         if (nxt_openssl_bundle_hash_insert(task, &conf->bundle_hash, item,
937                                            mp)
938             == NXT_ERROR)
939         {
940             return NXT_ERROR;
941         }
942     }
943 
944     return NXT_OK;
945 
946 fail:
947 
948     sk_GENERAL_NAME_pop_free(alt_names, GENERAL_NAME_free);
949 
950     return NXT_ERROR;
951 }
952 
953 
954 static const nxt_lvlhsh_proto_t  nxt_openssl_bundle_hash_proto
955     nxt_aligned(64) =
956 {
957     NXT_LVLHSH_DEFAULT,
958     nxt_openssl_bundle_hash_test,
959     nxt_mp_lvlhsh_alloc,
960     nxt_mp_lvlhsh_free,
961 };
962 
963 
964 static nxt_int_t
965 nxt_openssl_bundle_hash_test(nxt_lvlhsh_query_t *lhq, void *data)
966 {
967     nxt_tls_bundle_hash_item_t  *item;
968 
969     item = data;
970 
971     return nxt_strstr_eq(&lhq->key, &item->name) ? NXT_OK : NXT_DECLINED;
972 }
973 
974 
975 static nxt_int_t
976 nxt_openssl_bundle_hash_insert(nxt_task_t *task, nxt_lvlhsh_t *lvlhsh,
977     nxt_tls_bundle_hash_item_t *item, nxt_mp_t *mp)
978 {
979     nxt_str_t                   str;
980     nxt_int_t                   ret;
981     nxt_lvlhsh_query_t          lhq;
982     nxt_tls_bundle_hash_item_t  *old;
983 
984     str = item->name;
985 
986     if (item->name.start[0] == '*') {
987         item->name.start++;
988         item->name.length--;
989 
990         if (item->name.length == 0 || item->name.start[0] != '.') {
991             nxt_log(task, NXT_LOG_WARN, "ignored invalid name \"%V\" "
992                     "in certificate \"%V\": missing \".\" "
993                     "after wildcard symbol", &str, &item->bundle->name);
994             return NXT_OK;
995         }
996     }
997 
998     lhq.pool = mp;
999     lhq.key = item->name;
1000     lhq.value = item;
1001     lhq.proto = &nxt_openssl_bundle_hash_proto;
1002     lhq.replace = 0;
1003     lhq.key_hash = nxt_murmur_hash2(item->name.start, item->name.length);
1004 
1005     ret = nxt_lvlhsh_insert(lvlhsh, &lhq);
1006     if (nxt_fast_path(ret == NXT_OK)) {
1007         nxt_debug(task, "name \"%V\" for certificate \"%V\" is inserted",
1008                   &str, &item->bundle->name);
1009         return NXT_OK;
1010     }
1011 
1012     if (nxt_fast_path(ret == NXT_DECLINED)) {
1013         old = lhq.value;
1014         if (old->bundle != item->bundle) {
1015             nxt_log(task, NXT_LOG_WARN, "ignored duplicate name \"%V\" "
1016                     "in certificate \"%V\", identical name appears in \"%V\"",
1017                     &str, &old->bundle->name, &item->bundle->name);
1018 
1019             old->bundle = item->bundle;
1020         }
1021 
1022         return NXT_OK;
1023     }
1024 
1025     return NXT_ERROR;
1026 }
1027 
1028 
1029 static nxt_int_t
1030 nxt_openssl_servername(SSL *s, int *ad, void *arg)
1031 {
1032     nxt_str_t              str;
1033     nxt_uint_t             i;
1034     nxt_conn_t             *c;
1035     const char             *servername;
1036     nxt_tls_conf_t         *conf;
1037     nxt_openssl_conn_t     *tls;
1038     nxt_tls_bundle_conf_t  *bundle;
1039 
1040     c = SSL_get_ex_data(s, nxt_openssl_connection_index);
1041 
1042     if (nxt_slow_path(c == NULL)) {
1043         nxt_thread_log_alert("SSL_get_ex_data() failed");
1044         return SSL_TLSEXT_ERR_ALERT_FATAL;
1045     }
1046 
1047     servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name);
1048 
1049     if (servername == NULL) {
1050         nxt_debug(c->socket.task, "SSL_get_servername(): NULL");
1051         goto done;
1052     }
1053 
1054     str.length = nxt_strlen(servername);
1055     if (str.length == 0) {
1056         nxt_debug(c->socket.task, "SSL_get_servername(): \"\" is empty");
1057         goto done;
1058     }
1059 
1060     if (servername[0] == '.') {
1061         nxt_debug(c->socket.task, "ignored the server name \"%s\": "
1062                                   "leading \".\"", servername);
1063         goto done;
1064     }
1065 
1066     nxt_debug(c->socket.task, "tls with servername \"%s\"", servername);
1067 
1068     str.start = nxt_mp_nget(c->mem_pool, str.length);
1069     if (nxt_slow_path(str.start == NULL)) {
1070         return SSL_TLSEXT_ERR_ALERT_FATAL;
1071     }
1072 
1073     nxt_memcpy_lowcase(str.start, (const u_char *) servername, str.length);
1074 
1075     tls = c->u.tls;
1076     conf = tls->conf;
1077 
1078     bundle = nxt_openssl_find_ctx(conf, &str);
1079 
1080     if (bundle == NULL) {
1081         for (i = 1; i < str.length; i++) {
1082             if (str.start[i] == '.') {
1083                 str.start += i;
1084                 str.length -= i;
1085 
1086                 bundle = nxt_openssl_find_ctx(conf, &str);
1087                 break;
1088             }
1089         }
1090     }
1091 
1092     if (bundle != NULL) {
1093         nxt_debug(c->socket.task, "new tls context found for \"%V\": \"%V\" "
1094                                   "(old: \"%V\")", &str, &bundle->name,
1095                                   &conf->bundle->name);
1096 
1097         if (bundle != conf->bundle) {
1098             if (SSL_set_SSL_CTX(s, bundle->ctx) == NULL) {
1099                 nxt_openssl_log_error(c->socket.task, NXT_LOG_ALERT,
1100                                       "SSL_set_SSL_CTX() failed");
1101 
1102                 return SSL_TLSEXT_ERR_ALERT_FATAL;
1103             }
1104         }
1105     }
1106 
1107 done:
1108 
1109     return SSL_TLSEXT_ERR_OK;
1110 }
1111 
1112 
1113 static nxt_tls_bundle_conf_t *
1114 nxt_openssl_find_ctx(nxt_tls_conf_t *conf, nxt_str_t *sn)
1115 {
1116     nxt_int_t                   ret;
1117     nxt_lvlhsh_query_t          lhq;
1118     nxt_tls_bundle_hash_item_t  *item;
1119 
1120     lhq.key_hash = nxt_murmur_hash2(sn->start, sn->length);
1121     lhq.key = *sn;
1122     lhq.proto = &nxt_openssl_bundle_hash_proto;
1123 
1124     ret = nxt_lvlhsh_find(&conf->bundle_hash, &lhq);
1125     if (ret != NXT_OK) {
1126         return NULL;
1127     }
1128 
1129     item = lhq.value;
1130 
1131     return item->bundle;
1132 }
1133 
1134 
1135 static void
1136 nxt_openssl_server_free(nxt_task_t *task, nxt_tls_conf_t *conf)
1137 {
1138     nxt_tls_bundle_conf_t  *bundle;
1139 
1140     bundle = conf->bundle;
1141     nxt_assert(bundle != NULL);
1142 
1143     do {
1144         SSL_CTX_free(bundle->ctx);
1145         bundle = bundle->next;
1146     } while (bundle != NULL);
1147 
1148     if (conf->tickets) {
1149         nxt_memzero(conf->tickets->tickets,
1150                     conf->tickets->count * sizeof(nxt_tls_ticket_t));
1151     }
1152 
1153 #if (OPENSSL_VERSION_NUMBER >= 0x1010100fL \
1154      && OPENSSL_VERSION_NUMBER < 0x1010101fL)
1155     RAND_keep_random_devices_open(0);
1156 #endif
1157 }
1158 
1159 
1160 static void
1161 nxt_openssl_conn_init(nxt_task_t *task, nxt_tls_conf_t *conf, nxt_conn_t *c)
1162 {
1163     int                 ret;
1164     SSL                 *s;
1165     SSL_CTX             *ctx;
1166     nxt_openssl_conn_t  *tls;
1167 
1168     nxt_log_debug(c->socket.log, "openssl conn init");
1169 
1170     tls = nxt_mp_zget(c->mem_pool, sizeof(nxt_openssl_conn_t));
1171     if (tls == NULL) {
1172         goto fail;
1173     }
1174 
1175     c->u.tls = tls;
1176     nxt_buf_mem_set_size(&tls->buffer, conf->buffer_size);
1177 
1178     ctx = conf->bundle->ctx;
1179 
1180     s = SSL_new(ctx);
1181     if (s == NULL) {
1182         nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_new() failed");
1183         goto fail;
1184     }
1185 
1186     tls->session = s;
1187     /* To pass TLS config to the nxt_openssl_servername() callback. */
1188     tls->conf = conf;
1189     tls->conn = c;
1190 
1191     ret = SSL_set_fd(s, c->socket.fd);
1192 
1193     if (ret == 0) {
1194         nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_set_fd(%d) failed",
1195                               c->socket.fd);
1196         goto fail;
1197     }
1198 
1199     SSL_set_accept_state(s);
1200 
1201     if (SSL_set_ex_data(s, nxt_openssl_connection_index, c) == 0) {
1202         nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_set_ex_data() failed");
1203         goto fail;
1204     }
1205 
1206     c->io = &nxt_openssl_conn_io;
1207     c->sendfile = NXT_CONN_SENDFILE_OFF;
1208 
1209     nxt_openssl_conn_handshake(task, c, c->socket.data);
1210 
1211     return;
1212 
1213 fail:
1214 
1215     nxt_work_queue_add(c->read_work_queue, c->read_state->error_handler,
1216                        task, c, c->socket.data);
1217 }
1218 
1219 
1220 nxt_inline void
1221 nxt_openssl_conn_free(nxt_task_t *task, nxt_conn_t *c)
1222 {
1223     nxt_openssl_conn_t  *tls;
1224 
1225     nxt_debug(task, "openssl conn free");
1226 
1227     tls = c->u.tls;
1228 
1229     if (tls != NULL) {
1230         c->u.tls = NULL;
1231         nxt_free(tls->buffer.start);
1232         SSL_free(tls->session);
1233     }
1234 }
1235 
1236 
1237 static void
1238 nxt_openssl_conn_handshake(nxt_task_t *task, void *obj, void *data)
1239 {
1240     int                     ret;
1241     nxt_int_t               n;
1242     nxt_err_t               err;
1243     nxt_conn_t              *c;
1244     nxt_work_queue_t        *wq;
1245     nxt_work_handler_t      handler;
1246     nxt_openssl_conn_t      *tls;
1247     const nxt_conn_state_t  *state;
1248 
1249     c = obj;
1250 
1251     nxt_debug(task, "openssl conn handshake fd:%d", c->socket.fd);
1252 
1253     if (c->socket.error != 0) {
1254         return;
1255     }
1256 
1257     tls = c->u.tls;
1258 
1259     if (tls == NULL) {
1260         return;
1261     }
1262 
1263     nxt_debug(task, "openssl conn handshake: %d times", tls->times);
1264 
1265     /* "tls->times == 1" is suitable to run SSL_do_handshake() in job. */
1266 
1267     ret = SSL_do_handshake(tls->session);
1268 
1269     err = (ret <= 0) ? nxt_socket_errno : 0;
1270 
1271     nxt_thread_time_debug_update(task->thread);
1272 
1273     nxt_debug(task, "SSL_do_handshake(%d): %d err:%d", c->socket.fd, ret, err);
1274 
1275     state = (c->read_state != NULL) ? c->read_state : c->write_state;
1276 
1277     if (ret > 0) {
1278         /* ret == 1, the handshake was successfully completed. */
1279         tls->handshake = 1;
1280 
1281         if (c->read_state != NULL) {
1282             if (state->io_read_handler != NULL || c->read != NULL) {
1283                 nxt_conn_read(task->thread->engine, c);
1284                 return;
1285             }
1286 
1287         } else {
1288             if (c->write != NULL) {
1289                 nxt_conn_write(task->thread->engine, c);
1290                 return;
1291             }
1292         }
1293 
1294         handler = state->ready_handler;
1295 
1296     } else {
1297         c->socket.read_handler = nxt_openssl_conn_handshake;
1298         c->socket.write_handler = nxt_openssl_conn_handshake;
1299 
1300         n = nxt_openssl_conn_test_error(task, c, ret, err,
1301                                         NXT_OPENSSL_HANDSHAKE);
1302         switch (n) {
1303 
1304         case NXT_AGAIN:
1305             if (tls->ssl_error == SSL_ERROR_WANT_READ && tls->times < 2) {
1306                 tls->times++;
1307             }
1308 
1309             return;
1310 
1311         case 0:
1312             handler = state->close_handler;
1313             break;
1314 
1315         default:
1316         case NXT_ERROR:
1317             nxt_openssl_conn_error(task, err, "SSL_do_handshake(%d) failed",
1318                                    c->socket.fd);
1319 
1320             handler = state->error_handler;
1321             break;
1322         }
1323     }
1324 
1325     wq = (c->read_state != NULL) ? c->read_work_queue : c->write_work_queue;
1326 
1327     nxt_work_queue_add(wq, handler, task, c, data);
1328 }
1329 
1330 
1331 static ssize_t
1332 nxt_openssl_conn_io_recvbuf(nxt_conn_t *c, nxt_buf_t *b)
1333 {
1334     int                 ret;
1335     size_t              size;
1336     nxt_int_t           n;
1337     nxt_err_t           err;
1338     nxt_openssl_conn_t  *tls;
1339 
1340     tls = c->u.tls;
1341     size = b->mem.end - b->mem.free;
1342 
1343     ret = SSL_read(tls->session, b->mem.free, size);
1344 
1345     err = (ret <= 0) ? nxt_socket_errno : 0;
1346 
1347     nxt_debug(c->socket.task, "SSL_read(%d, %p, %uz): %d err:%d",
1348               c->socket.fd, b->mem.free, size, ret, err);
1349 
1350     if (ret > 0) {
1351         return ret;
1352     }
1353 
1354     n = nxt_openssl_conn_test_error(c->socket.task, c, ret, err,
1355                                     NXT_OPENSSL_READ);
1356     if (n == NXT_ERROR) {
1357         nxt_openssl_conn_error(c->socket.task, err,
1358                                "SSL_read(%d, %p, %uz) failed",
1359                                c->socket.fd, b->mem.free, size);
1360     }
1361 
1362     return n;
1363 }
1364 
1365 
1366 static ssize_t
1367 nxt_openssl_conn_io_sendbuf(nxt_task_t *task, nxt_sendbuf_t *sb)
1368 {
1369     nxt_uint_t    niov;
1370     struct iovec  iov;
1371 
1372     niov = nxt_sendbuf_mem_coalesce0(task, sb, &iov, 1);
1373 
1374     if (niov == 0 && sb->sync) {
1375         return 0;
1376     }
1377 
1378     return nxt_openssl_conn_io_send(task, sb, iov.iov_base, iov.iov_len);
1379 }
1380 
1381 
1382 static ssize_t
1383 nxt_openssl_conn_io_send(nxt_task_t *task, nxt_sendbuf_t *sb, void *buf,
1384     size_t size)
1385 {
1386     int                 ret;
1387     nxt_err_t           err;
1388     nxt_int_t           n;
1389     nxt_conn_t          *c;
1390     nxt_openssl_conn_t  *tls;
1391 
1392     tls = sb->tls;
1393 
1394     ret = SSL_write(tls->session, buf, size);
1395 
1396     err = (ret <= 0) ? nxt_socket_errno : 0;
1397 
1398     nxt_debug(task, "SSL_write(%d, %p, %uz): %d err:%d",
1399               sb->socket, buf, size, ret, err);
1400 
1401     if (ret > 0) {
1402         return ret;
1403     }
1404 
1405     c = tls->conn;
1406     c->socket.write_ready = sb->ready;
1407 
1408     n = nxt_openssl_conn_test_error(task, c, ret, err, NXT_OPENSSL_WRITE);
1409 
1410     sb->ready = c->socket.write_ready;
1411 
1412     if (n == NXT_ERROR) {
1413         sb->error = c->socket.error;
1414         nxt_openssl_conn_error(task, err, "SSL_write(%d, %p, %uz) failed",
1415                                sb->socket, buf, size);
1416     }
1417 
1418     return n;
1419 }
1420 
1421 
1422 static void
1423 nxt_openssl_conn_io_shutdown(nxt_task_t *task, void *obj, void *data)
1424 {
1425     int                 ret, mode;
1426     SSL                 *s;
1427     nxt_err_t           err;
1428     nxt_int_t           n;
1429     nxt_bool_t          quiet, once;
1430     nxt_conn_t          *c;
1431     nxt_openssl_conn_t  *tls;
1432     nxt_work_handler_t  handler;
1433 
1434     c = obj;
1435 
1436     nxt_debug(task, "openssl conn shutdown fd:%d", c->socket.fd);
1437 
1438     c->read_state = NULL;
1439     tls = c->u.tls;
1440 
1441     if (tls == NULL) {
1442         return;
1443     }
1444 
1445     s = tls->session;
1446 
1447     if (s == NULL || !tls->handshake) {
1448         handler = c->write_state->ready_handler;
1449         goto done;
1450     }
1451 
1452     mode = SSL_get_shutdown(s);
1453 
1454     if (c->socket.timedout || c->socket.error != 0) {
1455         quiet = 1;
1456 
1457     } else if (c->socket.closed && !(mode & SSL_RECEIVED_SHUTDOWN)) {
1458         quiet = 1;
1459 
1460     } else {
1461         quiet = 0;
1462     }
1463 
1464     SSL_set_quiet_shutdown(s, quiet);
1465 
1466     if (tls->conf->no_wait_shutdown) {
1467         mode |= SSL_RECEIVED_SHUTDOWN;
1468     }
1469 
1470     once = 1;
1471 
1472     for ( ;; ) {
1473         SSL_set_shutdown(s, mode);
1474 
1475         ret = SSL_shutdown(s);
1476 
1477         err = (ret <= 0) ? nxt_socket_errno : 0;
1478 
1479         nxt_debug(task, "SSL_shutdown(%d, %d, %b): %d err:%d",
1480                   c->socket.fd, mode, quiet, ret, err);
1481 
1482         if (ret > 0) {
1483             /* ret == 1, the shutdown was successfully completed. */
1484             handler = c->write_state->ready_handler;
1485             goto done;
1486         }
1487 
1488         if (ret == 0) {
1489             /*
1490              * If SSL_shutdown() returns 0 then it should be called
1491              * again.  The second SSL_shutdown() call should return
1492              * -1/SSL_ERROR_WANT_READ or -1/SSL_ERROR_WANT_WRITE.
1493              * OpenSSL prior to 0.9.8m version however never returns
1494              * -1 at all.  Fortunately, OpenSSL preserves internally
1495              * correct status available via SSL_get_error(-1).
1496              */
1497             if (once) {
1498                 once = 0;
1499                 mode = SSL_get_shutdown(s);
1500                 continue;
1501             }
1502 
1503             ret = -1;
1504         }
1505 
1506         /* ret == -1 */
1507 
1508         break;
1509     }
1510 
1511     c->socket.read_handler = nxt_openssl_conn_io_shutdown;
1512     c->socket.write_handler = nxt_openssl_conn_io_shutdown;
1513     c->socket.error_handler = c->write_state->error_handler;
1514 
1515     n = nxt_openssl_conn_test_error(task, c, ret, err, NXT_OPENSSL_SHUTDOWN);
1516 
1517     switch (n) {
1518 
1519     case 0:
1520         handler = c->write_state->close_handler;
1521         break;
1522 
1523     case NXT_AGAIN:
1524         c->write_timer.handler = nxt_openssl_conn_io_shutdown_timeout;
1525         nxt_timer_add(task->thread->engine, &c->write_timer, 5000);
1526         return;
1527 
1528     default:
1529     case NXT_ERROR:
1530         nxt_openssl_conn_error(task, err, "SSL_shutdown(%d) failed",
1531                                c->socket.fd);
1532         handler = c->write_state->error_handler;
1533     }
1534 
1535 done:
1536 
1537     nxt_openssl_conn_free(task, c);
1538 
1539     nxt_work_queue_add(c->write_work_queue, handler, task, c, data);
1540 }
1541 
1542 
1543 static nxt_int_t
1544 nxt_openssl_conn_test_error(nxt_task_t *task, nxt_conn_t *c, int ret,
1545     nxt_err_t sys_err, nxt_openssl_io_t io)
1546 {
1547     u_long              lib_err;
1548     nxt_openssl_conn_t  *tls;
1549 
1550     tls = c->u.tls;
1551 
1552     tls->ssl_error = SSL_get_error(tls->session, ret);
1553 
1554     nxt_debug(task, "SSL_get_error(): %d", tls->ssl_error);
1555 
1556     switch (tls->ssl_error) {
1557 
1558     case SSL_ERROR_WANT_READ:
1559         c->socket.read_ready = 0;
1560 
1561         if (io != NXT_OPENSSL_READ) {
1562             nxt_fd_event_block_write(task->thread->engine, &c->socket);
1563 
1564             if (nxt_fd_event_is_disabled(c->socket.read)) {
1565                 nxt_fd_event_enable_read(task->thread->engine, &c->socket);
1566             }
1567         }
1568 
1569         return NXT_AGAIN;
1570 
1571     case SSL_ERROR_WANT_WRITE:
1572         c->socket.write_ready = 0;
1573 
1574         if (io != NXT_OPENSSL_WRITE) {
1575             nxt_fd_event_block_read(task->thread->engine, &c->socket);
1576 
1577             if (nxt_fd_event_is_disabled(c->socket.write)) {
1578                 nxt_fd_event_enable_write(task->thread->engine, &c->socket);
1579             }
1580         }
1581 
1582         return NXT_AGAIN;
1583 
1584     case SSL_ERROR_SYSCALL:
1585         lib_err = ERR_peek_error();
1586 
1587         nxt_debug(task, "ERR_peek_error(): %l", lib_err);
1588 
1589         if (sys_err != 0 || lib_err != 0) {
1590             c->socket.error = sys_err;
1591             return NXT_ERROR;
1592         }
1593 
1594         /* A connection was just closed. */
1595         c->socket.closed = 1;
1596         return 0;
1597 
1598     case SSL_ERROR_ZERO_RETURN:
1599         /* A "close notify" alert. */
1600         return 0;
1601 
1602     default: /* SSL_ERROR_SSL, etc. */
1603         c->socket.error = 1000;  /* Nonexistent errno code. */
1604         return NXT_ERROR;
1605     }
1606 }
1607 
1608 
1609 static void
1610 nxt_openssl_conn_io_shutdown_timeout(nxt_task_t *task, void *obj, void *data)
1611 {
1612     nxt_conn_t   *c;
1613     nxt_timer_t  *timer;
1614 
1615     timer = obj;
1616 
1617     nxt_debug(task, "openssl conn shutdown timeout");
1618 
1619     c = nxt_write_timer_conn(timer);
1620 
1621     c->socket.timedout = 1;
1622     nxt_openssl_conn_io_shutdown(task, c, NULL);
1623 }
1624 
1625 
1626 static void nxt_cdecl
1627 nxt_openssl_conn_error(nxt_task_t *task, nxt_err_t err, const char *fmt, ...)
1628 {
1629     u_char      *p, *end;
1630     va_list     args;
1631     nxt_uint_t  level;
1632     u_char      msg[NXT_MAX_ERROR_STR];
1633 
1634     level = nxt_openssl_log_error_level(err);
1635 
1636     if (nxt_log_level_enough(task->log, level)) {
1637 
1638         end = msg + sizeof(msg);
1639 
1640         va_start(args, fmt);
1641         p = nxt_vsprintf(msg, end, fmt, args);
1642         va_end(args);
1643 
1644         if (err != 0) {
1645             p = nxt_sprintf(p, end, " %E", err);
1646         }
1647 
1648         p = nxt_openssl_copy_error(p, end);
1649 
1650         nxt_log(task, level, "%*s", p - msg, msg);
1651 
1652     } else {
1653         ERR_clear_error();
1654     }
1655 }
1656 
1657 
1658 static nxt_uint_t
1659 nxt_openssl_log_error_level(nxt_err_t err)
1660 {
1661     switch (ERR_GET_REASON(ERR_peek_error())) {
1662 
1663     case 0:
1664         return nxt_socket_error_level(err);
1665 
1666     case SSL_R_BAD_CHANGE_CIPHER_SPEC:                    /*  103 */
1667     case SSL_R_BLOCK_CIPHER_PAD_IS_WRONG:                 /*  129 */
1668     case SSL_R_DIGEST_CHECK_FAILED:                       /*  149 */
1669     case SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST:             /*  151 */
1670     case SSL_R_EXCESSIVE_MESSAGE_SIZE:                    /*  152 */
1671     case SSL_R_LENGTH_MISMATCH:                           /*  159 */
1672 #ifdef SSL_R_NO_CIPHERS_PASSED
1673     case SSL_R_NO_CIPHERS_PASSED:                         /*  182 */
1674 #endif
1675     case SSL_R_NO_CIPHERS_SPECIFIED:                      /*  183 */
1676     case SSL_R_NO_COMPRESSION_SPECIFIED:                  /*  187 */
1677     case SSL_R_NO_SHARED_CIPHER:                          /*  193 */
1678     case SSL_R_RECORD_LENGTH_MISMATCH:                    /*  213 */
1679 #ifdef SSL_R_PARSE_TLSEXT
1680     case SSL_R_PARSE_TLSEXT:                              /*  227 */
1681 #endif
1682     case SSL_R_UNEXPECTED_MESSAGE:                        /*  244 */
1683     case SSL_R_UNEXPECTED_RECORD:                         /*  245 */
1684     case SSL_R_UNKNOWN_ALERT_TYPE:                        /*  246 */
1685     case SSL_R_UNKNOWN_PROTOCOL:                          /*  252 */
1686     case SSL_R_WRONG_VERSION_NUMBER:                      /*  267 */
1687     case SSL_R_DECRYPTION_FAILED_OR_BAD_RECORD_MAC:       /*  281 */
1688 #ifdef SSL_R_RENEGOTIATE_EXT_TOO_LONG
1689     case SSL_R_RENEGOTIATE_EXT_TOO_LONG:                  /*  335 */
1690     case SSL_R_RENEGOTIATION_ENCODING_ERR:                /*  336 */
1691     case SSL_R_RENEGOTIATION_MISMATCH:                    /*  337 */
1692 #endif
1693 #ifdef SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED
1694     case SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED:      /*  338 */
1695 #endif
1696 #ifdef SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING
1697     case SSL_R_SCSV_RECEIVED_WHEN_RENEGOTIATING:          /*  345 */
1698 #endif
1699     case 1000:/* SSL_R_SSLV3_ALERT_CLOSE_NOTIFY */
1700     case SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE:            /* 1010 */
1701     case SSL_R_SSLV3_ALERT_BAD_RECORD_MAC:                /* 1020 */
1702     case SSL_R_TLSV1_ALERT_DECRYPTION_FAILED:             /* 1021 */
1703     case SSL_R_TLSV1_ALERT_RECORD_OVERFLOW:               /* 1022 */
1704     case SSL_R_SSLV3_ALERT_DECOMPRESSION_FAILURE:         /* 1030 */
1705     case SSL_R_SSLV3_ALERT_HANDSHAKE_FAILURE:             /* 1040 */
1706     case SSL_R_SSLV3_ALERT_ILLEGAL_PARAMETER:             /* 1047 */
1707         break;
1708 
1709     case SSL_R_SSLV3_ALERT_NO_CERTIFICATE:                /* 1041 */
1710     case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE:               /* 1042 */
1711     case SSL_R_SSLV3_ALERT_UNSUPPORTED_CERTIFICATE:       /* 1043 */
1712     case SSL_R_SSLV3_ALERT_CERTIFICATE_REVOKED:           /* 1044 */
1713     case SSL_R_SSLV3_ALERT_CERTIFICATE_EXPIRED:           /* 1045 */
1714     case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN:           /* 1046 */
1715     case SSL_R_TLSV1_ALERT_UNKNOWN_CA:                    /* 1048 */
1716     case SSL_R_TLSV1_ALERT_ACCESS_DENIED:                 /* 1049 */
1717     case SSL_R_TLSV1_ALERT_DECODE_ERROR:                  /* 1050 */
1718     case SSL_R_TLSV1_ALERT_DECRYPT_ERROR:                 /* 1051 */
1719     case SSL_R_TLSV1_ALERT_EXPORT_RESTRICTION:            /* 1060 */
1720     case SSL_R_TLSV1_ALERT_PROTOCOL_VERSION:              /* 1070 */
1721     case SSL_R_TLSV1_ALERT_INSUFFICIENT_SECURITY:         /* 1071 */
1722     case SSL_R_TLSV1_ALERT_INTERNAL_ERROR:                /* 1080 */
1723     case SSL_R_TLSV1_ALERT_USER_CANCELLED:                /* 1090 */
1724     case SSL_R_TLSV1_ALERT_NO_RENEGOTIATION:              /* 1100 */
1725         return NXT_LOG_ERR;
1726 
1727     default:
1728         return NXT_LOG_ALERT;
1729     }
1730 
1731     return NXT_LOG_INFO;
1732 }
1733 
1734 
1735 void nxt_cdecl
1736 nxt_openssl_log_error(nxt_task_t *task, nxt_uint_t level, const char *fmt, ...)
1737 {
1738     u_char   *p, *end;
1739     va_list  args;
1740     u_char   msg[NXT_MAX_ERROR_STR];
1741 
1742     end = msg + sizeof(msg);
1743 
1744     va_start(args, fmt);
1745     p = nxt_vsprintf(msg, end, fmt, args);
1746     va_end(args);
1747 
1748     p = nxt_openssl_copy_error(p, end);
1749 
1750     nxt_log(task, level, "%*s", p - msg, msg);
1751 }
1752 
1753 
1754 u_char *
1755 nxt_openssl_copy_error(u_char *p, u_char *end)
1756 {
1757     int         flags;
1758     u_long      err;
1759     nxt_bool_t  clear;
1760     const char  *data, *delimiter;
1761 
1762     err = ERR_peek_error();
1763     if (err == 0) {
1764         return p;
1765     }
1766 
1767     /* Log the most relevant error message ... */
1768     data = ERR_reason_error_string(err);
1769 
1770     p = nxt_sprintf(p, end, " (%d: %s) (OpenSSL: ", ERR_GET_REASON(err), data);
1771 
1772     /*
1773      * ... followed by all queued cumbersome OpenSSL error messages
1774      * and drain the error queue.
1775      */
1776     delimiter = "";
1777     clear = 0;
1778 
1779     for ( ;; ) {
1780 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
1781         err = ERR_get_error_all(NULL, NULL, NULL, &data, &flags);
1782 #else
1783         err = ERR_get_error_line_data(NULL, NULL, &data, &flags);
1784 #endif
1785         if (err == 0) {
1786             break;
1787         }
1788 
1789         p = nxt_sprintf(p, end, "%s", delimiter);
1790 
1791         ERR_error_string_n(err, (char *) p, end - p);
1792 
1793         while (p < end && *p != '\0') {
1794             p++;
1795         }
1796 
1797         if ((flags & ERR_TXT_STRING) != 0) {
1798             p = nxt_sprintf(p, end, ":%s", data);
1799         }
1800 
1801         clear |= ((flags & ERR_TXT_MALLOCED) != 0);
1802 
1803         delimiter = "; ";
1804     }
1805 
1806     /* Deallocate additional data. */
1807 
1808     if (clear) {
1809         ERR_clear_error();
1810     }
1811 
1812     if (p < end) {
1813         *p++ = ')';
1814     }
1815 
1816     return p;
1817 }
1818