1 /** @file
2 
3   A brief file description
4 
5   @section license License
6 
7   Licensed to the Apache Software Foundation (ASF) under one
8   or more contributor license agreements.  See the NOTICE file
9   distributed with this work for additional information
10   regarding copyright ownership.  The ASF licenses this file
11   to you under the Apache License, Version 2.0 (the
12   "License"); you may not use this file except in compliance
13   with the License.  You may obtain a copy of the License at
14 
15       http://www.apache.org/licenses/LICENSE-2.0
16 
17   Unless required by applicable law or agreed to in writing, software
18   distributed under the License is distributed on an "AS IS" BASIS,
19   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20   See the License for the specific language governing permissions and
21   limitations under the License.
22  */
23 
24 #include "tscore/ink_config.h"
25 #include "tscore/EventNotify.h"
26 #include "tscore/I_Layout.h"
27 #include "tscore/TSSystemState.h"
28 
29 #include "InkAPIInternal.h" // Added to include the ssl_hook definitions
30 #include "Log.h"
31 #include "HttpTunnel.h"
32 #include "ProxyProtocol.h"
33 #include "HttpConfig.h"
34 
35 #include "P_Net.h"
36 #include "P_SSLUtils.h"
37 #include "P_SSLNextProtocolSet.h"
38 #include "P_SSLConfig.h"
39 #include "P_SSLClientUtils.h"
40 #include "P_SSLSNI.h"
41 #include "BIO_fastopen.h"
42 #include "SSLStats.h"
43 #include "SSLInternal.h"
44 #include "P_ALPNSupport.h"
45 
46 #include <climits>
47 #include <string>
48 #include <cstring>
49 
50 using namespace std::literals;
51 
52 #if TS_USE_TLS_ASYNC
53 #include <openssl/async.h>
54 #endif
55 
56 // This is missing from BoringSSL
57 #ifndef BIO_eof
58 #define BIO_eof(b) (int)BIO_ctrl(b, BIO_CTRL_EOF, 0, nullptr)
59 #endif
60 
61 #define SSL_READ_ERROR_NONE 0
62 #define SSL_READ_ERROR 1
63 #define SSL_READ_READY 2
64 #define SSL_READ_COMPLETE 3
65 #define SSL_READ_WOULD_BLOCK 4
66 #define SSL_READ_EOS 5
67 #define SSL_HANDSHAKE_WANT_READ 6
68 #define SSL_HANDSHAKE_WANT_WRITE 7
69 #define SSL_HANDSHAKE_WANT_ACCEPT 8
70 #define SSL_HANDSHAKE_WANT_CONNECT 9
71 #define SSL_WRITE_WOULD_BLOCK 10
72 #define SSL_WAIT_FOR_HOOK 11
73 #define SSL_WAIT_FOR_ASYNC 12
74 
75 ClassAllocator<SSLNetVConnection> sslNetVCAllocator("sslNetVCAllocator");
76 
77 namespace
78 {
79 /// Callback to get two locks.
80 /// The lock for this continuation, and for the target continuation.
81 class ContWrapper : public Continuation
82 {
83 public:
84   /** Constructor.
85       This takes the secondary @a mutex and the @a target continuation
86       to invoke, along with the arguments for that invocation.
87   */
ContWrapper(ProxyMutex * mutex,Continuation * target,int eventId=EVENT_IMMEDIATE,void * edata=nullptr)88   ContWrapper(ProxyMutex *mutex,             ///< Mutex for this continuation (primary lock).
89               Continuation *target,          ///< "Real" continuation we want to call.
90               int eventId = EVENT_IMMEDIATE, ///< Event ID for invocation of @a target.
91               void *edata = nullptr          ///< Data for invocation of @a target.
92               )
93     : Continuation(mutex), _target(target), _eventId(eventId), _edata(edata)
94   {
95     SET_HANDLER(&ContWrapper::event_handler);
96   }
97 
98   /// Required event handler method.
99   int
event_handler(int,void *)100   event_handler(int, void *)
101   {
102     EThread *eth = this_ethread();
103 
104     MUTEX_TRY_LOCK(lock, _target->mutex, eth);
105     if (lock.is_locked()) { // got the target lock, we can proceed.
106       _target->handleEvent(_eventId, _edata);
107       delete this;
108     } else { // can't get both locks, try again.
109       eventProcessor.schedule_imm(this, ET_NET);
110     }
111     return 0;
112   }
113 
114   /** Convenience static method.
115 
116       This lets a client make one call and not have to (accurately)
117       copy the invocation logic embedded here. We duplicate it near
118       by textually so it is easier to keep in sync.
119 
120       This takes the same arguments as the constructor but, if the
121       lock can be obtained immediately, does not construct an
122       instance but simply calls the @a target.
123   */
124   static void
wrap(ProxyMutex * mutex,Continuation * target,int eventId=EVENT_IMMEDIATE,void * edata=nullptr)125   wrap(ProxyMutex *mutex,             ///< Mutex for this continuation (primary lock).
126        Continuation *target,          ///< "Real" continuation we want to call.
127        int eventId = EVENT_IMMEDIATE, ///< Event ID for invocation of @a target.
128        void *edata = nullptr          ///< Data for invocation of @a target.
129   )
130   {
131     EThread *eth = this_ethread();
132     if (!target->mutex) {
133       // If there's no mutex, plugin doesn't care about locking so why should we?
134       target->handleEvent(eventId, edata);
135     } else {
136       MUTEX_TRY_LOCK(lock, target->mutex, eth);
137       if (lock.is_locked()) {
138         target->handleEvent(eventId, edata);
139       } else {
140         eventProcessor.schedule_imm(new ContWrapper(mutex, target, eventId, edata), ET_NET);
141       }
142     }
143   }
144 
145 private:
146   Continuation *_target; ///< Continuation to invoke.
147   int _eventId;          ///< with this event
148   void *_edata;          ///< and this data
149 };
150 } // namespace
151 
152 //
153 // Private
154 //
155 
156 static SSL *
make_ssl_connection(SSL_CTX * ctx,SSLNetVConnection * netvc)157 make_ssl_connection(SSL_CTX *ctx, SSLNetVConnection *netvc)
158 {
159   SSL *ssl;
160 
161   if (likely(ssl = SSL_new(ctx))) {
162     netvc->ssl = ssl;
163 
164     // Only set up the bio stuff for the server side
165     if (netvc->get_context() == NET_VCONNECTION_OUT) {
166       BIO *bio = BIO_new(const_cast<BIO_METHOD *>(BIO_s_fastopen()));
167       BIO_set_fd(bio, netvc->get_socket(), BIO_NOCLOSE);
168 
169       if (netvc->options.f_tcp_fastopen) {
170         BIO_set_conn_address(bio, netvc->get_remote_addr());
171       }
172 
173       SSL_set_bio(ssl, bio, bio);
174     } else {
175       netvc->initialize_handshake_buffers();
176       BIO *rbio = BIO_new(BIO_s_mem());
177       BIO *wbio = BIO_new_fd(netvc->get_socket(), BIO_NOCLOSE);
178       BIO_set_mem_eof_return(wbio, -1);
179       SSL_set_bio(ssl, rbio, wbio);
180 
181 #if TS_HAS_TLS_EARLY_DATA
182       // Must disable OpenSSL's internal anti-replay if external cache is used with
183       // 0-rtt, otherwise session reuse will be broken. The freshness check described
184       // in https://tools.ietf.org/html/rfc8446#section-8.3 is still performed. But we
185       // still need to implement something to try to prevent replay atacks.
186       //
187       // We are now also disabling this when using OpenSSL's internal cache, since we
188       // are calling "ssl_accept" non-blocking, it seems to be confusing the anti-replay
189       // mechanism and causing session resumption to fail.
190       SSLConfig::scoped_config params;
191       if (SSL_version(ssl) >= TLS1_3_VERSION && params->server_max_early_data > 0) {
192         bool ret1 = false;
193         bool ret2 = false;
194         if ((ret1 = SSL_set_max_early_data(ssl, params->server_max_early_data)) == 1) {
195           Debug("ssl_early_data", "SSL_set_max_early_data: success");
196         } else {
197           Debug("ssl_early_data", "SSL_set_max_early_data: failed");
198         }
199 
200         if ((ret2 = SSL_set_recv_max_early_data(ssl, params->server_recv_max_early_data)) == 1) {
201           Debug("ssl_early_data", "SSL_set_recv_max_early_data: success");
202         } else {
203           Debug("ssl_early_data", "SSL_set_recv_max_early_data: failed");
204         }
205 
206         if (ret1 && ret2) {
207           Debug("ssl_early_data", "Must disable anti-replay if 0-rtt is enabled.");
208           SSL_set_options(ssl, SSL_OP_NO_ANTI_REPLAY);
209         }
210       }
211 #endif
212     }
213 
214     SSLNetVCAttach(ssl, netvc);
215     TLSSessionResumptionSupport::bind(ssl, netvc);
216   }
217 
218   return ssl;
219 }
220 
221 static void
debug_certificate_name(const char * msg,X509_NAME * name)222 debug_certificate_name(const char *msg, X509_NAME *name)
223 {
224   BIO *bio;
225 
226   if (name == nullptr) {
227     return;
228   }
229 
230   bio = BIO_new(BIO_s_mem());
231   if (bio == nullptr) {
232     return;
233   }
234 
235   if (X509_NAME_print_ex(bio, name, 0 /* indent */, XN_FLAG_ONELINE) > 0) {
236     long len;
237     char *ptr;
238     len = BIO_get_mem_data(bio, &ptr);
239     Debug("ssl", "%s %.*s", msg, (int)len, ptr);
240   }
241 
242   BIO_free(bio);
243 }
244 
245 static int
ssl_read_from_net(SSLNetVConnection * sslvc,EThread * lthread,int64_t & ret)246 ssl_read_from_net(SSLNetVConnection *sslvc, EThread *lthread, int64_t &ret)
247 {
248   NetState *s            = &sslvc->read;
249   MIOBufferAccessor &buf = s->vio.buffer;
250   int event              = SSL_READ_ERROR_NONE;
251   int64_t bytes_read     = 0;
252   ssl_error_t sslErr     = SSL_ERROR_NONE;
253 
254   int64_t toread = buf.writer()->write_avail();
255   ink_release_assert(toread > 0);
256   if (toread > s->vio.ntodo()) {
257     toread = s->vio.ntodo();
258   }
259 
260   bytes_read = 0;
261   while (sslErr == SSL_ERROR_NONE && bytes_read < toread) {
262     int64_t nread             = 0;
263     int64_t block_write_avail = buf.writer()->block_write_avail();
264     ink_release_assert(block_write_avail > 0);
265     int64_t amount_to_read = toread - bytes_read;
266     if (amount_to_read > block_write_avail) {
267       amount_to_read = block_write_avail;
268     }
269 
270     Debug("ssl", "amount_to_read=%" PRId64, amount_to_read);
271     char *current_block = buf.writer()->end();
272     ink_release_assert(current_block != nullptr);
273     sslErr = SSLReadBuffer(sslvc->ssl, current_block, amount_to_read, nread);
274 
275     Debug("ssl", "nread=%" PRId64, nread);
276 
277     switch (sslErr) {
278     case SSL_ERROR_NONE:
279 
280 #if DEBUG
281       SSLDebugBufferPrint("ssl_buff", current_block, nread, "SSL Read");
282 #endif
283       ink_assert(nread);
284       bytes_read += nread;
285       if (nread > 0) {
286         buf.writer()->fill(nread); // Tell the buffer, we've used the bytes
287         sslvc->netActivity(lthread);
288       }
289       break;
290     case SSL_ERROR_WANT_WRITE:
291       event = SSL_WRITE_WOULD_BLOCK;
292       Debug("ssl.error", "SSL_ERROR_WOULD_BLOCK(write)");
293       break;
294     case SSL_ERROR_WANT_READ:
295       event = SSL_READ_WOULD_BLOCK;
296       Debug("ssl.error", "SSL_ERROR_WOULD_BLOCK(read)");
297       break;
298 #ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
299     case SSL_ERROR_WANT_CLIENT_HELLO_CB:
300       event = SSL_READ_WOULD_BLOCK;
301       Debug("ssl.error", "SSL_ERROR_WOULD_BLOCK(read/client hello cb)");
302       break;
303 #endif
304     case SSL_ERROR_WANT_X509_LOOKUP:
305       event = SSL_READ_WOULD_BLOCK;
306       Debug("ssl.error", "SSL_ERROR_WOULD_BLOCK(read/x509 lookup)");
307       break;
308     case SSL_ERROR_SYSCALL:
309       SSL_INCREMENT_DYN_STAT(ssl_error_syscall);
310       if (nread != 0) {
311         // not EOF
312         event = SSL_READ_ERROR;
313         ret   = errno;
314         Debug("ssl.error", "SSL_ERROR_SYSCALL, underlying IO error: %s", strerror(errno));
315       } else {
316         // then EOF observed, treat it as EOS
317         // Error("[SSL_NetVConnection::ssl_read_from_net] SSL_ERROR_SYSCALL, EOF observed violating SSL protocol");
318         event = SSL_READ_EOS;
319       }
320       break;
321     case SSL_ERROR_ZERO_RETURN:
322       event = SSL_READ_EOS;
323       Debug("ssl.error", "SSL_ERROR_ZERO_RETURN");
324       break;
325     case SSL_ERROR_SSL:
326     default: {
327       char buf[512];
328       unsigned long e = ERR_peek_last_error();
329       ERR_error_string_n(e, buf, sizeof(buf));
330       event = SSL_READ_ERROR;
331       ret   = errno;
332       SSL_CLR_ERR_INCR_DYN_STAT(sslvc, ssl_error_ssl, "errno=%d", errno);
333     } break;
334     } // switch
335   }   // while
336 
337   if (bytes_read > 0) {
338     Debug("ssl", "bytes_read=%" PRId64, bytes_read);
339 
340     s->vio.ndone += bytes_read;
341     sslvc->netActivity(lthread);
342 
343     ret = bytes_read;
344 
345     // If we read it all, don't worry about the other events and just send read complete
346     event = (s->vio.ntodo() <= 0) ? SSL_READ_COMPLETE : SSL_READ_READY;
347     if (sslErr == SSL_ERROR_NONE && s->vio.ntodo() > 0) {
348       // We stopped with data on the wire (to avoid overbuffering).  Make sure we are triggered
349       sslvc->read.triggered = 1;
350     }
351   } else { // if( bytes_read > 0 )
352 #if defined(_DEBUG)
353     if (bytes_read == 0) {
354       Debug("ssl", "bytes_read == 0");
355     }
356 #endif
357   }
358   return event;
359 }
360 
361 /** Read from socket directly for handshake data.  Store the data in an MIOBuffer.  Place the data in
362  * the read BIO so the openssl library has access to it. If for some reason we must abort out of the
363  * handshake, the stored data can be replayed (e.g. back out to blind tunneling)
364  */
365 int64_t
read_raw_data()366 SSLNetVConnection::read_raw_data()
367 {
368   // read data
369   int64_t r          = 0;
370   int64_t total_read = 0;
371   int64_t rattempted = 0;
372   char *buffer       = nullptr;
373   int buf_len;
374   IOBufferBlock *b = this->handShakeBuffer->first_write_block();
375 
376   rattempted = b->write_avail();
377   while (rattempted) {
378     buffer  = b->_end;
379     buf_len = rattempted;
380     b       = b->next.get();
381 
382     r = socketManager.read(this->con.fd, buffer, buf_len);
383     NET_INCREMENT_DYN_STAT(net_calls_to_read_stat);
384     total_read += rattempted;
385 
386     Debug("ssl", "read_raw_data r=%" PRId64 " rattempted=%" PRId64 " total_read=%" PRId64 " fd=%d", r, rattempted, total_read,
387           con.fd);
388     // last read failed or was incomplete
389     if (r != rattempted || !b) {
390       break;
391     }
392 
393     rattempted = b->write_avail();
394   }
395   // If we have already moved some bytes successfully, adjust total_read to reflect reality
396   // If any read succeeded, we should return success
397   if (r != rattempted) {
398     // If the first read fails, we should return error
399     if (r <= 0 && total_read > rattempted) {
400       r = total_read - rattempted;
401     } else {
402       r = total_read - rattempted + r;
403     }
404   }
405   NET_SUM_DYN_STAT(net_read_bytes_stat, r);
406 
407   IpMap *pp_ipmap;
408   pp_ipmap = SSLConfigParams::proxy_protocol_ipmap;
409 
410   if (this->get_is_proxy_protocol()) {
411     Debug("proxyprotocol", "proxy protocol is enabled on this port");
412     if (pp_ipmap->count() > 0) {
413       Debug("proxyprotocol", "proxy protocol has a configured allowlist of trusted IPs - checking");
414 
415       // At this point, using get_remote_addr() will return the ip of the
416       // proxy source IP, not the Proxy Protocol client ip. Since we are
417       // checking the ip of the actual source of this connection, this is
418       // what we want now.
419       void *payload = nullptr;
420       if (!pp_ipmap->contains(get_remote_addr(), &payload)) {
421         Debug("proxyprotocol", "proxy protocol src IP is NOT in the configured allowlist of trusted IPs - "
422                                "closing connection");
423         r = -ENOTCONN; // Need a quick close/exit here to refuse the connection!!!!!!!!!
424         goto proxy_protocol_bypass;
425       } else {
426         char new_host[INET6_ADDRSTRLEN];
427         Debug("proxyprotocol", "Source IP [%s] is in the trusted allowlist for proxy protocol",
428               ats_ip_ntop(this->get_remote_addr(), new_host, sizeof(new_host)));
429       }
430     } else {
431       Debug("proxyprotocol", "proxy protocol DOES NOT have a configured allowlist of trusted IPs but "
432                              "proxy protocol is enabled on this port - processing all connections");
433     }
434 
435     if (ssl_has_proxy_v1(this, buffer, &r)) {
436       Debug("proxyprotocol", "ssl has proxy_v1 header");
437       set_remote_addr(get_proxy_protocol_src_addr());
438     } else {
439       Debug("proxyprotocol", "proxy protocol was enabled, but required header was not present in the "
440                              "transaction - closing connection");
441     }
442   } // end of Proxy Protocol processing
443 
444 proxy_protocol_bypass:
445 
446   if (r > 0) {
447     this->handShakeBuffer->fill(r);
448 
449     char *start              = this->handShakeReader->start();
450     char *end                = this->handShakeReader->end();
451     this->handShakeBioStored = end - start;
452 
453     // Sets up the buffer as a read only bio target
454     // Must be reset on each read
455     BIO *rbio = BIO_new_mem_buf(start, this->handShakeBioStored);
456     BIO_set_mem_eof_return(rbio, -1);
457     SSL_set0_rbio(this->ssl, rbio);
458   } else {
459     this->handShakeBioStored = 0;
460   }
461 
462   Debug("ssl", "%p read r=%" PRId64 " total=%" PRId64 " bio=%d\n", this, r, total_read, this->handShakeBioStored);
463 
464   // check for errors
465   if (r <= 0) {
466     if (r == -EAGAIN || r == -ENOTCONN) {
467       NET_INCREMENT_DYN_STAT(net_calls_to_read_nodata_stat);
468     }
469   }
470 
471   return r;
472 }
473 
474 //
475 // Return true if we updated the rbio with another
476 // memory chunk (should be ready for another read right away)
477 //
478 bool
update_rbio(bool move_to_socket)479 SSLNetVConnection::update_rbio(bool move_to_socket)
480 {
481   bool retval = false;
482   if (BIO_eof(SSL_get_rbio(this->ssl)) && this->handShakeReader != nullptr) {
483     this->handShakeReader->consume(this->handShakeBioStored);
484     this->handShakeBioStored = 0;
485     // Load up the next block if present
486     if (this->handShakeReader->is_read_avail_more_than(0)) {
487       // Setup the next iobuffer block to drain
488       char *start              = this->handShakeReader->start();
489       char *end                = this->handShakeReader->end();
490       this->handShakeBioStored = end - start;
491 
492       // Sets up the buffer as a read only bio target
493       // Must be reset on each read
494       BIO *rbio = BIO_new_mem_buf(start, this->handShakeBioStored);
495       BIO_set_mem_eof_return(rbio, -1);
496       SSL_set0_rbio(this->ssl, rbio);
497       retval = true;
498       // Handshake buffer is empty but we have read something, move to the socket rbio
499     } else if (move_to_socket && this->handShakeHolder->is_read_avail_more_than(0)) {
500       BIO *rbio = BIO_new_fd(this->get_socket(), BIO_NOCLOSE);
501       BIO_set_mem_eof_return(rbio, -1);
502       SSL_set0_rbio(this->ssl, rbio);
503       free_handshake_buffers();
504     }
505   }
506   return retval;
507 }
508 
509 // changed by YTS Team, yamsat
510 void
net_read_io(NetHandler * nh,EThread * lthread)511 SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread)
512 {
513   int ret;
514   int64_t r     = 0;
515   int64_t bytes = 0;
516   NetState *s   = &this->read;
517 
518   if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == this->attributes) {
519     this->super::net_read_io(nh, lthread);
520     return;
521   }
522 
523   MUTEX_TRY_LOCK(lock, s->vio.mutex, lthread);
524   if (!lock.is_locked()) {
525     readReschedule(nh);
526     return;
527   }
528   // Got closed by the HttpSessionManager thread during a migration
529   // The closed flag should be stable once we get the s->vio.mutex in that case
530   // (the global session pool mutex).
531   if (this->closed) {
532     this->super::net_read_io(nh, lthread);
533     return;
534   }
535   // If the key renegotiation failed it's over, just signal the error and finish.
536   if (sslClientRenegotiationAbort == true) {
537     this->read.triggered = 0;
538     readSignalError(nh, static_cast<int>(r));
539     Debug("ssl", "client renegotiation setting read signal error");
540     return;
541   }
542 
543   // If it is not enabled, lower its priority.  This allows
544   // a fast connection to speed match a slower connection by
545   // shifting down in priority even if it could read.
546   if (!s->enabled || s->vio.op != VIO::READ || s->vio.is_disabled()) {
547     read_disable(nh, this);
548     return;
549   }
550 
551   MIOBufferAccessor &buf = s->vio.buffer;
552   int64_t ntodo          = s->vio.ntodo();
553   ink_assert(buf.writer());
554 
555   // Continue on if we are still in the handshake
556   if (!getSSLHandShakeComplete()) {
557     int err = 0;
558 
559     if (get_context() == NET_VCONNECTION_OUT) {
560       ret = sslStartHandShake(SSL_EVENT_CLIENT, err);
561     } else {
562       ret = sslStartHandShake(SSL_EVENT_SERVER, err);
563     }
564     // If we have flipped to blind tunnel, don't read ahead
565     if (this->handShakeReader) {
566       if (this->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL) {
567         // Now in blind tunnel. Set things up to read what is in the buffer
568         // Must send the READ_COMPLETE here before considering
569         // forwarding on the handshake buffer, so the
570         // SSLNextProtocolTrampoline has a chance to do its
571         // thing before forwarding the buffers.
572         this->readSignalDone(VC_EVENT_READ_COMPLETE, nh);
573 
574         // If the handshake isn't set yet, this means the tunnel
575         // decision was make in the SNI callback.  We must move
576         // the client hello message back into the standard read.vio
577         // so it will get forwarded onto the origin server
578         if (!this->getSSLHandShakeComplete()) {
579           this->sslHandshakeStatus = SSL_HANDSHAKE_DONE;
580 
581           // Copy over all data already read in during the SSL_accept
582           // (the client hello message)
583           NetState *s            = &this->read;
584           MIOBufferAccessor &buf = s->vio.buffer;
585           int64_t r              = buf.writer()->write(this->handShakeHolder);
586           s->vio.nbytes += r;
587           s->vio.ndone += r;
588 
589           // Clean up the handshake buffers
590           this->free_handshake_buffers();
591 
592           if (r > 0) {
593             // Kick things again, so the data that was copied into the
594             // vio.read buffer gets processed
595             this->readSignalDone(VC_EVENT_READ_COMPLETE, nh);
596           }
597         }
598         return; // Leave if we are tunneling
599       }
600     }
601     if (ret == EVENT_ERROR) {
602       this->read.triggered = 0;
603       readSignalError(nh, err);
604     } else if (ret == SSL_HANDSHAKE_WANT_READ || ret == SSL_HANDSHAKE_WANT_ACCEPT) {
605       if (SSLConfigParams::ssl_handshake_timeout_in > 0) {
606         double handshake_time = (static_cast<double>(Thread::get_hrtime() - sslHandshakeBeginTime) / 1000000000);
607         Debug("ssl", "ssl handshake for vc %p, took %.3f seconds, configured handshake_timer: %d", this, handshake_time,
608               SSLConfigParams::ssl_handshake_timeout_in);
609         if (handshake_time > SSLConfigParams::ssl_handshake_timeout_in) {
610           Debug("ssl", "ssl handshake for vc %p, expired, release the connection", this);
611           read.triggered = 0;
612           nh->read_ready_list.remove(this);
613           readSignalError(nh, VC_EVENT_EOS);
614           return;
615         }
616       }
617       // move over to the socket if we haven't already
618       if (this->handShakeBuffer != nullptr) {
619         read.triggered = update_rbio(true);
620       } else {
621         read.triggered = 0;
622       }
623       if (!read.triggered) {
624         nh->read_ready_list.remove(this);
625       }
626       readReschedule(nh);
627     } else if (ret == SSL_HANDSHAKE_WANT_CONNECT || ret == SSL_HANDSHAKE_WANT_WRITE) {
628       write.triggered = 0;
629       nh->write_ready_list.remove(this);
630       writeReschedule(nh);
631     } else if (ret == EVENT_DONE) {
632       Debug("ssl", "ssl handshake EVENT_DONE ntodo=%" PRId64, ntodo);
633       // If this was driven by a zero length read, signal complete when
634       // the handshake is complete. Otherwise set up for continuing read
635       // operations.
636       if (ntodo <= 0) {
637         readSignalDone(VC_EVENT_READ_COMPLETE, nh);
638       } else {
639         read.triggered = 1;
640         if (read.enabled) {
641           nh->read_ready_list.in_or_enqueue(this);
642         }
643       }
644     } else if (ret == SSL_WAIT_FOR_HOOK || ret == SSL_WAIT_FOR_ASYNC) {
645       // avoid readReschedule - done when the plugin calls us back to reenable
646     } else {
647       readReschedule(nh);
648     }
649     return;
650   }
651 
652   // If there is nothing to do or no space available, disable connection
653   if (ntodo <= 0 || !buf.writer()->write_avail() || s->vio.is_disabled()) {
654     read_disable(nh, this);
655     return;
656   }
657 
658   // At this point we are at the post-handshake SSL processing
659   //
660   // not sure if this do-while loop is really needed here, please replace
661   // this comment if you know
662   do {
663     ret = ssl_read_from_net(this, lthread, r);
664     if (ret == SSL_READ_READY || ret == SSL_READ_ERROR_NONE) {
665       bytes += r;
666     }
667     ink_assert(bytes >= 0);
668   } while ((ret == SSL_READ_READY && bytes == 0) || ret == SSL_READ_ERROR_NONE);
669 
670   if (bytes > 0) {
671     if (ret == SSL_READ_WOULD_BLOCK || ret == SSL_READ_READY) {
672       if (readSignalAndUpdate(VC_EVENT_READ_READY) != EVENT_CONT) {
673         Debug("ssl", "readSignal != EVENT_CONT");
674         return;
675       }
676     }
677   }
678 
679   switch (ret) {
680   case SSL_READ_READY:
681     readReschedule(nh);
682     return;
683     break;
684   case SSL_WRITE_WOULD_BLOCK:
685   case SSL_READ_WOULD_BLOCK:
686     if (lock.get_mutex() != s->vio.mutex.get()) {
687       Debug("ssl", "mutex switched");
688       if (ret == SSL_READ_WOULD_BLOCK) {
689         readReschedule(nh);
690       } else {
691         writeReschedule(nh);
692       }
693       return;
694     }
695     // reset the trigger and remove from the ready queue
696     // we will need to be retriggered to read from this socket again
697     read.triggered = 0;
698     nh->read_ready_list.remove(this);
699     Debug("ssl", "read finished - would block");
700 #if TS_USE_PORT
701     if (ret == SSL_READ_WOULD_BLOCK) {
702       readReschedule(nh);
703     } else {
704       writeReschedule(nh);
705     }
706 #endif
707     break;
708 
709   case SSL_READ_EOS:
710     // close the connection if we have SSL_READ_EOS, this is the return value from ssl_read_from_net() if we get an
711     // SSL_ERROR_ZERO_RETURN from SSL_get_error()
712     // SSL_ERROR_ZERO_RETURN means that the origin server closed the SSL connection
713     read.triggered = 0;
714     readSignalDone(VC_EVENT_EOS, nh);
715 
716     if (bytes > 0) {
717       Debug("ssl", "read finished - EOS");
718     } else {
719       Debug("ssl", "read finished - 0 useful bytes read, bytes used by SSL layer");
720     }
721     break;
722   case SSL_READ_COMPLETE:
723     readSignalDone(VC_EVENT_READ_COMPLETE, nh);
724     Debug("ssl", "read finished - signal done");
725     break;
726   case SSL_READ_ERROR:
727     this->read.triggered = 0;
728     readSignalError(nh, static_cast<int>(r));
729     Debug("ssl", "read finished - read error");
730     break;
731   }
732 }
733 
734 int64_t
load_buffer_and_write(int64_t towrite,MIOBufferAccessor & buf,int64_t & total_written,int & needs)735 SSLNetVConnection::load_buffer_and_write(int64_t towrite, MIOBufferAccessor &buf, int64_t &total_written, int &needs)
736 {
737   int64_t try_to_write;
738   int64_t num_really_written       = 0;
739   int64_t l                        = 0;
740   uint32_t dynamic_tls_record_size = 0;
741   ssl_error_t err                  = SSL_ERROR_NONE;
742 
743   // Dynamic TLS record sizing
744   ink_hrtime now = 0;
745   if (SSLConfigParams::ssl_maxrecord == -1) {
746     now                       = Thread::get_hrtime_updated();
747     int msec_since_last_write = ink_hrtime_diff_msec(now, sslLastWriteTime);
748 
749     if (msec_since_last_write > SSL_DEF_TLS_RECORD_MSEC_THRESHOLD) {
750       // reset sslTotalBytesSent upon inactivity for SSL_DEF_TLS_RECORD_MSEC_THRESHOLD
751       sslTotalBytesSent = 0;
752     }
753     Debug("ssl", "now=%" PRId64 " lastwrite=%" PRId64 " msec_since_last_write=%d", now, sslLastWriteTime, msec_since_last_write);
754   }
755 
756   if (HttpProxyPort::TRANSPORT_BLIND_TUNNEL == this->attributes) {
757     return this->super::load_buffer_and_write(towrite, buf, total_written, needs);
758   }
759 
760   Debug("ssl", "towrite=%" PRId64, towrite);
761 
762   do {
763     // What is remaining left in the next block?
764     l                   = buf.reader()->block_read_avail();
765     char *current_block = buf.reader()->start();
766 
767     // check if to amount to write exceeds that in this buffer
768     int64_t wavail = towrite - total_written;
769 
770     if (l > wavail) {
771       l = wavail;
772     }
773 
774     // TS-2365: If the SSL max record size is set and we have
775     // more data than that, break this into smaller write
776     // operations.
777     //
778     // TS-4424: Don't mess with record size if last SSL_write failed with
779     // needs write
780     if (redoWriteSize) {
781       l             = redoWriteSize;
782       redoWriteSize = 0;
783     } else {
784       if (SSLConfigParams::ssl_maxrecord > 0 && l > SSLConfigParams::ssl_maxrecord) {
785         l = SSLConfigParams::ssl_maxrecord;
786       } else if (SSLConfigParams::ssl_maxrecord == -1) {
787         if (sslTotalBytesSent < SSL_DEF_TLS_RECORD_BYTE_THRESHOLD) {
788           dynamic_tls_record_size = SSL_DEF_TLS_RECORD_SIZE;
789           SSL_INCREMENT_DYN_STAT(ssl_total_dyn_def_tls_record_count);
790         } else {
791           dynamic_tls_record_size = SSL_MAX_TLS_RECORD_SIZE;
792           SSL_INCREMENT_DYN_STAT(ssl_total_dyn_max_tls_record_count);
793         }
794         if (l > dynamic_tls_record_size) {
795           l = dynamic_tls_record_size;
796         }
797       }
798     }
799 
800     if (!l) {
801       break;
802     }
803 
804     try_to_write       = l;
805     num_really_written = 0;
806     Debug("v_ssl", "b=%p l=%" PRId64, current_block, l);
807     err = SSLWriteBuffer(ssl, current_block, l, num_really_written);
808 
809     // We wrote all that we thought we should
810     if (num_really_written > 0) {
811       total_written += num_really_written;
812       buf.reader()->consume(num_really_written);
813     }
814 
815     Debug("ssl", "try_to_write=%" PRId64 " written=%" PRId64 " total_written=%" PRId64, try_to_write, num_really_written,
816           total_written);
817     NET_INCREMENT_DYN_STAT(net_calls_to_write_stat);
818   } while (num_really_written == try_to_write && total_written < towrite);
819 
820   if (total_written > 0) {
821     sslLastWriteTime = now;
822     sslTotalBytesSent += total_written;
823   }
824   redoWriteSize = 0;
825   if (num_really_written > 0) {
826     needs |= EVENTIO_WRITE;
827   } else {
828     switch (err) {
829     case SSL_ERROR_NONE:
830       Debug("ssl", "SSL_write-SSL_ERROR_NONE");
831       break;
832     case SSL_ERROR_WANT_READ:
833       needs |= EVENTIO_READ;
834       num_really_written = -EAGAIN;
835       Debug("ssl.error", "SSL_write-SSL_ERROR_WANT_READ");
836       break;
837     case SSL_ERROR_WANT_WRITE:
838 #ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
839     case SSL_ERROR_WANT_CLIENT_HELLO_CB:
840 #endif
841     case SSL_ERROR_WANT_X509_LOOKUP: {
842       if (SSL_ERROR_WANT_WRITE == err) {
843         redoWriteSize = l;
844       }
845       needs |= EVENTIO_WRITE;
846       num_really_written = -EAGAIN;
847       Debug("ssl.error", "SSL_write-SSL_ERROR_WANT_WRITE");
848       break;
849     }
850     case SSL_ERROR_SYSCALL:
851       num_really_written = -errno;
852       SSL_INCREMENT_DYN_STAT(ssl_error_syscall);
853       Debug("ssl.error", "SSL_write-SSL_ERROR_SYSCALL");
854       break;
855     // end of stream
856     case SSL_ERROR_ZERO_RETURN:
857       num_really_written = -errno;
858       Debug("ssl.error", "SSL_write-SSL_ERROR_ZERO_RETURN");
859       break;
860     case SSL_ERROR_SSL:
861     default: {
862       // Treat SSL_ERROR_SSL as EPIPE error.
863       num_really_written = -EPIPE;
864       SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSL_write-SSL_ERROR_SSL errno=%d", errno);
865     } break;
866     }
867   }
868   return num_really_written;
869 }
870 
SSLNetVConnection()871 SSLNetVConnection::SSLNetVConnection() {}
872 
873 void
do_io_close(int lerrno)874 SSLNetVConnection::do_io_close(int lerrno)
875 {
876   if (this->ssl != nullptr) {
877     if (get_context() == NET_VCONNECTION_OUT) {
878       callHooks(TS_EVENT_VCONN_OUTBOUND_CLOSE);
879     } else {
880       callHooks(TS_EVENT_VCONN_CLOSE);
881     }
882 
883     if (getSSLHandShakeComplete()) {
884       int shutdown_mode = SSL_get_shutdown(ssl);
885       Debug("ssl-shutdown", "previous shutdown state 0x%x", shutdown_mode);
886       int new_shutdown_mode = shutdown_mode | SSL_RECEIVED_SHUTDOWN;
887 
888       if (new_shutdown_mode != shutdown_mode) {
889         // We do not need to sit around and wait for the client's close-notify if
890         // they have not already sent it.  We will still be standards compliant
891         Debug("ssl-shutdown", "new SSL_set_shutdown 0x%x", new_shutdown_mode);
892         SSL_set_shutdown(ssl, new_shutdown_mode);
893       }
894 
895       // If the peer has already sent a FIN, don't bother with the shutdown
896       // They will just send us a RST for our troubles
897       // This test is not foolproof.  The client's fin could be on the wire
898       // at the same time we send the close-notify.  If so, the client will likely
899       // send RST anyway
900       char c;
901       ssize_t x = recv(this->con.fd, &c, 1, MSG_PEEK);
902       // x < 0 means error.  x == 0 means fin sent
903       bool do_shutdown = (x > 0);
904       if (x < 0) {
905         do_shutdown = (errno == EAGAIN || errno == EWOULDBLOCK);
906       }
907       if (do_shutdown) {
908         // Send the close-notify
909         int ret = SSL_shutdown(ssl);
910         Debug("ssl-shutdown", "SSL_shutdown %s", (ret) ? "success" : "failed");
911       } else {
912         // Request a quiet shutdown to OpenSSL
913         SSL_set_quiet_shutdown(ssl, 1);
914         SSL_set_shutdown(ssl, SSL_RECEIVED_SHUTDOWN | SSL_SENT_SHUTDOWN);
915         Debug("ssl-shutdown", "Enable quiet shutdown");
916       }
917     }
918   }
919   // Go on and do the unix socket cleanups
920   super::do_io_close(lerrno);
921 }
922 
923 void
clear()924 SSLNetVConnection::clear()
925 {
926   _serverName.reset();
927 
928   if (ssl != nullptr) {
929     SSL_free(ssl);
930     ssl = nullptr;
931   }
932   ALPNSupport::clear();
933   TLSSessionResumptionSupport::clear();
934 
935   sslHandshakeStatus          = SSL_HANDSHAKE_ONGOING;
936   sslHandshakeBeginTime       = 0;
937   sslLastWriteTime            = 0;
938   sslTotalBytesSent           = 0;
939   sslClientRenegotiationAbort = false;
940 
941   curHook         = nullptr;
942   hookOpRequested = SSL_HOOK_OP_DEFAULT;
943   free_handshake_buffers();
944 
945   super::clear();
946 }
947 void
free(EThread * t)948 SSLNetVConnection::free(EThread *t)
949 {
950   ink_release_assert(t == this_ethread());
951 
952   // cancel OOB
953   cancel_OOB();
954   // close socket fd
955   if (con.fd != NO_FD) {
956     NET_SUM_GLOBAL_DYN_STAT(net_connections_currently_open_stat, -1);
957   }
958   con.close();
959 
960   ats_free(tunnel_host);
961 
962   if (early_data_reader != nullptr) {
963     early_data_reader->dealloc();
964   }
965 
966   if (early_data_buf != nullptr) {
967     free_MIOBuffer(early_data_buf);
968   }
969 
970   early_data_reader = nullptr;
971   early_data_buf    = nullptr;
972 
973   clear();
974   SET_CONTINUATION_HANDLER(this, (SSLNetVConnHandler)&SSLNetVConnection::startEvent);
975   ink_assert(con.fd == NO_FD);
976   ink_assert(t == this_ethread());
977 
978   if (from_accept_thread) {
979     sslNetVCAllocator.free(this);
980   } else {
981     ink_assert(con.fd == NO_FD);
982     THREAD_FREE(this, sslNetVCAllocator, t);
983   }
984 }
985 
986 int
sslStartHandShake(int event,int & err)987 SSLNetVConnection::sslStartHandShake(int event, int &err)
988 {
989   if (TSSystemState::is_ssl_handshaking_stopped()) {
990     Debug("ssl", "Stopping handshake due to server shutting down.");
991     return EVENT_ERROR;
992   }
993   if (sslHandshakeBeginTime == 0) {
994     sslHandshakeBeginTime = Thread::get_hrtime();
995     // net_activity will not be triggered until after the handshake
996     set_inactivity_timeout(HRTIME_SECONDS(SSLConfigParams::ssl_handshake_timeout_in));
997   }
998   SSLConfig::scoped_config params;
999   switch (event) {
1000   case SSL_EVENT_SERVER:
1001     if (this->ssl == nullptr) {
1002       SSLCertificateConfig::scoped_config lookup;
1003       IpEndpoint dst;
1004       int namelen = sizeof(dst);
1005       if (0 != safe_getsockname(this->get_socket(), &dst.sa, &namelen)) {
1006         Debug("ssl", "Failed to get dest ip, errno = [%d]", errno);
1007         return EVENT_ERROR;
1008       }
1009       SSLCertContext *cc = lookup->find(dst);
1010       if (is_debug_tag_set("ssl")) {
1011         IpEndpoint src;
1012         ip_port_text_buffer ipb1, ipb2;
1013         int ip_len = sizeof(src);
1014 
1015         if (0 != safe_getpeername(this->get_socket(), &src.sa, &ip_len)) {
1016           Debug("ssl", "Failed to get src ip, errno = [%d]", errno);
1017           return EVENT_ERROR;
1018         }
1019         ats_ip_nptop(&dst, ipb1, sizeof(ipb1));
1020         ats_ip_nptop(&src, ipb2, sizeof(ipb2));
1021         Debug("ssl", "IP context is %p for [%s] -> [%s], default context %p", cc, ipb2, ipb1, lookup->defaultContext());
1022       }
1023 
1024       // Escape if this is marked to be a tunnel.
1025       // No data has been read at this point, so we can go
1026       // directly into blind tunnel mode
1027 
1028       if (cc && SSLCertContextOption::OPT_TUNNEL == cc->opt) {
1029         if (this->is_transparent) {
1030           this->attributes   = HttpProxyPort::TRANSPORT_BLIND_TUNNEL;
1031           sslHandshakeStatus = SSL_HANDSHAKE_DONE;
1032           SSL_free(this->ssl);
1033           this->ssl = nullptr;
1034           return EVENT_DONE;
1035         } else {
1036           hookOpRequested = SSL_HOOK_OP_TUNNEL;
1037         }
1038       }
1039 
1040       // Attach the default SSL_CTX to this SSL session. The default context is never going to be able
1041       // to negotiate a SSL session, but it's enough to trampoline us into the SNI callback where we
1042       // can select the right server certificate.
1043       this->ssl = make_ssl_connection(lookup->defaultContext(), this);
1044     }
1045 
1046     if (this->ssl == nullptr) {
1047       SSLErrorVC(this, "failed to create SSL server session");
1048       return EVENT_ERROR;
1049     }
1050     return sslServerHandShakeEvent(err);
1051 
1052   case SSL_EVENT_CLIENT:
1053 
1054     char buff[INET6_ADDRSTRLEN];
1055 
1056     if (this->ssl == nullptr) {
1057       // Making the check here instead of later, so we only
1058       // do this setting immediately after we create the SSL object
1059       SNIConfig::scoped_config sniParam;
1060       const char *serverKey = this->options.sni_servername;
1061       if (!serverKey) {
1062         ats_ip_ntop(this->get_remote_addr(), buff, INET6_ADDRSTRLEN);
1063         serverKey = buff;
1064       }
1065       auto nps                 = sniParam->getPropertyConfig(serverKey);
1066       shared_SSL_CTX sharedCTX = nullptr;
1067       SSL_CTX *clientCTX       = nullptr;
1068 
1069       // First Look to see if there are override parameters
1070       if (options.ssl_client_cert_name) {
1071         std::string certFilePath = Layout::get()->relative_to(params->clientCertPathOnly, options.ssl_client_cert_name);
1072         std::string keyFilePath;
1073         if (options.ssl_client_private_key_name) {
1074           keyFilePath = Layout::get()->relative_to(params->clientKeyPathOnly, options.ssl_client_private_key_name);
1075         }
1076         std::string caCertFilePath;
1077         if (options.ssl_client_ca_cert_name) {
1078           caCertFilePath = Layout::get()->relative_to(params->clientCACertPath, options.ssl_client_ca_cert_name);
1079         }
1080         sharedCTX =
1081           params->getCTX(certFilePath.c_str(), keyFilePath.empty() ? nullptr : keyFilePath.c_str(),
1082                          caCertFilePath.empty() ? params->clientCACertFilename : caCertFilePath.c_str(), params->clientCACertPath);
1083       } else if (options.ssl_client_ca_cert_name) {
1084         std::string caCertFilePath = Layout::get()->relative_to(params->clientCACertPath, options.ssl_client_ca_cert_name);
1085         sharedCTX = params->getCTX(params->clientCertPath, params->clientKeyPath, caCertFilePath.c_str(), params->clientCACertPath);
1086       } else if (nps && !nps->client_cert_file.empty()) {
1087         // If no overrides available, try the available nextHopProperty by reading from context mappings
1088         sharedCTX =
1089           params->getCTX(nps->client_cert_file.c_str(), nps->client_key_file.empty() ? nullptr : nps->client_key_file.c_str(),
1090                          params->clientCACertFilename, params->clientCACertPath);
1091       } else { // Just stay with the values passed down from the SM for verify
1092         clientCTX = params->client_ctx.get();
1093       }
1094 
1095       if (sharedCTX) {
1096         clientCTX = sharedCTX.get();
1097       }
1098 
1099       if (options.verifyServerPolicy != YamlSNIConfig::Policy::UNSET) {
1100         // Stay with conf-override version as the highest priority
1101       } else if (nps && nps->verifyServerPolicy != YamlSNIConfig::Policy::UNSET) {
1102         options.verifyServerPolicy = nps->verifyServerPolicy;
1103       } else {
1104         options.verifyServerPolicy = params->verifyServerPolicy;
1105       }
1106 
1107       if (options.verifyServerProperties != YamlSNIConfig::Property::UNSET) {
1108         // Stay with conf-override version as the highest priority
1109       } else if (nps && nps->verifyServerProperties != YamlSNIConfig::Property::UNSET) {
1110         options.verifyServerProperties = nps->verifyServerProperties;
1111       } else {
1112         options.verifyServerProperties = params->verifyServerProperties;
1113       }
1114 
1115       if (!clientCTX) {
1116         SSLErrorVC(this, "failed to create SSL client session");
1117         return EVENT_ERROR;
1118       }
1119 
1120       this->ssl = make_ssl_connection(clientCTX, this);
1121       if (this->ssl == nullptr) {
1122         SSLErrorVC(this, "failed to create SSL client session");
1123         return EVENT_ERROR;
1124       }
1125 
1126       SSL_set_verify(this->ssl, SSL_VERIFY_PEER, verify_callback);
1127 
1128       ats_scoped_str &tlsext_host_name = this->options.sni_hostname ? this->options.sni_hostname : this->options.sni_servername;
1129       if (tlsext_host_name) {
1130         if (SSL_set_tlsext_host_name(this->ssl, tlsext_host_name)) {
1131           Debug("ssl", "using SNI name '%s' for client handshake", tlsext_host_name.get());
1132         } else {
1133           Debug("ssl.error", "failed to set SNI name '%s' for client handshake", tlsext_host_name.get());
1134           SSL_INCREMENT_DYN_STAT(ssl_sni_name_set_failure);
1135         }
1136       }
1137     }
1138 
1139     return sslClientHandShakeEvent(err);
1140 
1141   default:
1142     ink_assert(0);
1143     return EVENT_ERROR;
1144   }
1145 }
1146 
1147 int
sslServerHandShakeEvent(int & err)1148 SSLNetVConnection::sslServerHandShakeEvent(int &err)
1149 {
1150   // Continue on if we are in the invoked state.  The hook has not yet reenabled
1151   if (sslHandshakeHookState == HANDSHAKE_HOOKS_CERT_INVOKE || sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE ||
1152       sslHandshakeHookState == HANDSHAKE_HOOKS_PRE_INVOKE || sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE) {
1153     return SSL_WAIT_FOR_HOOK;
1154   }
1155 
1156   // Go do the preaccept hooks
1157   if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) {
1158     SSL_INCREMENT_DYN_STAT(ssl_total_attempts_handshake_count_in_stat);
1159     if (!curHook) {
1160       Debug("ssl", "Initialize preaccept curHook from NULL");
1161       curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_START_HOOK));
1162     } else {
1163       curHook = curHook->next();
1164     }
1165     // If no more hooks, move onto CLIENT HELLO
1166 
1167     if (nullptr == curHook) {
1168       sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO;
1169     } else {
1170       sslHandshakeHookState = HANDSHAKE_HOOKS_PRE_INVOKE;
1171       ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_START, this);
1172       return SSL_WAIT_FOR_HOOK;
1173     }
1174   }
1175 
1176   // If a blind tunnel was requested in the pre-accept calls, convert.
1177   // Again no data has been exchanged, so we can go directly
1178   // without data replay.
1179   // Note we can't arrive here if a hook is active.
1180 
1181   if (SSL_HOOK_OP_TUNNEL == hookOpRequested) {
1182     this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL;
1183     SSL_free(this->ssl);
1184     this->ssl = nullptr;
1185     // Don't mark the handshake as complete yet,
1186     // Will be checking for that flag not being set after
1187     // we get out of this callback, and then will shuffle
1188     // over the buffered handshake packets to the O.S.
1189     return EVENT_DONE;
1190   } else if (SSL_HOOK_OP_TERMINATE == hookOpRequested) {
1191     sslHandshakeStatus = SSL_HANDSHAKE_DONE;
1192     return EVENT_DONE;
1193   }
1194 
1195   Debug("ssl", "Go on with the handshake state=%d", sslHandshakeHookState);
1196 
1197   // All the pre-accept hooks have completed, proceed with the actual accept.
1198   if (this->handShakeReader) {
1199     if (BIO_eof(SSL_get_rbio(this->ssl))) { // No more data in the buffer
1200       // Is this the first read?
1201       if (!this->handShakeReader->is_read_avail_more_than(0) && !this->handShakeHolder->is_read_avail_more_than(0)) {
1202         Debug("ssl", "%p first read\n", this);
1203         // Read from socket to fill in the BIO buffer with the
1204         // raw handshake data before calling the ssl accept calls.
1205         int retval = this->read_raw_data();
1206         if (retval < 0) {
1207           if (retval == -EAGAIN) {
1208             // No data at the moment, hang tight
1209             SSLVCDebug(this, "SSL handshake: EAGAIN");
1210             return SSL_HANDSHAKE_WANT_READ;
1211           } else {
1212             // An error, make us go away
1213             SSLVCDebug(this, "SSL handshake error: read_retval=%d", retval);
1214             return EVENT_ERROR;
1215           }
1216         } else if (retval == 0) {
1217           // EOF, go away, we stopped in the handshake
1218           SSLVCDebug(this, "SSL handshake error: EOF");
1219           return EVENT_ERROR;
1220         }
1221       } else {
1222         update_rbio(false);
1223       }
1224     } // Still data in the BIO
1225   }
1226 
1227 #if TS_USE_TLS_ASYNC
1228   if (SSLConfigParams::async_handshake_enabled) {
1229     SSL_set_mode(ssl, SSL_MODE_ASYNC);
1230   }
1231 #endif
1232   ssl_error_t ssl_error = SSLAccept(ssl);
1233 #if TS_USE_TLS_ASYNC
1234   if (ssl_error == SSL_ERROR_WANT_ASYNC) {
1235     size_t numfds;
1236     OSSL_ASYNC_FD *waitfds;
1237     // Set up the epoll entry for the signalling
1238     if (SSL_get_all_async_fds(ssl, nullptr, &numfds) && numfds > 0) {
1239       // Allocate space for the waitfd on the stack, should only be one most all of the time
1240       waitfds = reinterpret_cast<OSSL_ASYNC_FD *>(alloca(sizeof(OSSL_ASYNC_FD) * numfds));
1241       if (SSL_get_all_async_fds(ssl, waitfds, &numfds) && numfds > 0) {
1242         // Temporarily disable regular net
1243         this->read.triggered  = false;
1244         this->write.triggered = false;
1245         this->ep.stop(); // Modify used in read_disable doesn't work for edge triggered epol
1246         // Have to have the read NetState enabled because we are using it for the signal vc
1247         read.enabled       = true;
1248         PollDescriptor *pd = get_PollDescriptor(this_ethread());
1249         this->ep.start(pd, waitfds[0], static_cast<NetEvent *>(this), EVENTIO_READ);
1250         this->ep.type = EVENTIO_READWRITE_VC;
1251       }
1252     }
1253   } else if (SSLConfigParams::async_handshake_enabled) {
1254     // Clean up the epoll entry for signalling
1255     SSL_clear_mode(ssl, SSL_MODE_ASYNC);
1256     this->ep.stop();
1257     // Reactivate the socket, ready to rock
1258     PollDescriptor *pd = get_PollDescriptor(this_ethread());
1259     this->ep.start(
1260       pd, this,
1261       EVENTIO_READ |
1262         EVENTIO_WRITE); // Again we must muck with the eventloop directly because of limits with these methods and edge trigger
1263     if (ssl_error == SSL_ERROR_WANT_READ) {
1264       this->reenable(&read.vio);
1265       this->read.triggered = 1;
1266     }
1267   }
1268 #endif
1269   if (ssl_error != SSL_ERROR_NONE) {
1270     err = errno;
1271     SSLVCDebug(this, "SSL handshake error: %s (%d), errno=%d", SSLErrorName(ssl_error), ssl_error, err);
1272 
1273     // start a blind tunnel if tr-pass is set and data does not look like ClientHello
1274     char *buf = handShakeBuffer ? handShakeBuffer->buf() : nullptr;
1275     if (getTransparentPassThrough() && buf && *buf != SSL_OP_HANDSHAKE) {
1276       SSLVCDebug(this, "Data does not look like SSL handshake, starting blind tunnel");
1277       this->attributes   = HttpProxyPort::TRANSPORT_BLIND_TUNNEL;
1278       sslHandshakeStatus = SSL_HANDSHAKE_ONGOING;
1279       return EVENT_CONT;
1280     }
1281   }
1282 
1283   switch (ssl_error) {
1284   case SSL_ERROR_NONE:
1285     if (is_debug_tag_set("ssl")) {
1286       X509 *cert = SSL_get_peer_certificate(ssl);
1287 
1288       Debug("ssl", "SSL server handshake completed successfully");
1289       if (cert) {
1290         debug_certificate_name("client certificate subject CN is", X509_get_subject_name(cert));
1291         debug_certificate_name("client certificate issuer CN is", X509_get_issuer_name(cert));
1292         X509_free(cert);
1293       }
1294     }
1295 
1296     sslHandshakeStatus = SSL_HANDSHAKE_DONE;
1297 
1298     if (sslHandshakeBeginTime) {
1299       sslHandshakeEndTime                 = Thread::get_hrtime();
1300       const ink_hrtime ssl_handshake_time = sslHandshakeEndTime - sslHandshakeBeginTime;
1301 
1302       Debug("ssl", "ssl handshake time:%" PRId64, ssl_handshake_time);
1303       SSL_INCREMENT_DYN_STAT_EX(ssl_total_handshake_time_stat, ssl_handshake_time);
1304       SSL_INCREMENT_DYN_STAT(ssl_total_success_handshake_count_in_stat);
1305     }
1306     {
1307       const unsigned char *proto = nullptr;
1308       unsigned len               = 0;
1309 
1310       increment_ssl_version_metric(SSL_version(ssl));
1311 
1312       // If it's possible to negotiate both NPN and ALPN, then ALPN
1313       // is preferred since it is the server's preference.  The server
1314       // preference would not be meaningful if we let the client
1315       // preference have priority.
1316       SSL_get0_alpn_selected(ssl, &proto, &len);
1317       if (len == 0) {
1318         SSL_get0_next_proto_negotiated(ssl, &proto, &len);
1319       }
1320 
1321       if (len) {
1322         if (!this->setSelectedProtocol(proto, len)) {
1323           return EVENT_ERROR;
1324         }
1325         Debug("ssl", "client selected next protocol '%.*s'", len, proto);
1326       } else {
1327         Debug("ssl", "client did not select a next protocol");
1328       }
1329     }
1330 
1331     return EVENT_DONE;
1332 
1333   case SSL_ERROR_WANT_CONNECT:
1334     return SSL_HANDSHAKE_WANT_CONNECT;
1335 
1336   case SSL_ERROR_WANT_WRITE:
1337     return SSL_HANDSHAKE_WANT_WRITE;
1338 
1339   case SSL_ERROR_WANT_READ:
1340     return SSL_HANDSHAKE_WANT_READ;
1341 #ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
1342   case SSL_ERROR_WANT_CLIENT_HELLO_CB:
1343     return EVENT_CONT;
1344 #endif
1345 // This value is only defined in openssl has been patched to
1346 // enable the sni callback to break out of the SSL_accept processing
1347 #ifdef SSL_ERROR_WANT_SNI_RESOLVE
1348   case SSL_ERROR_WANT_X509_LOOKUP:
1349     return EVENT_CONT;
1350   case SSL_ERROR_WANT_SNI_RESOLVE:
1351 #elif SSL_ERROR_WANT_X509_LOOKUP
1352   case SSL_ERROR_WANT_X509_LOOKUP:
1353 #endif
1354 #if defined(SSL_ERROR_WANT_SNI_RESOLVE) || defined(SSL_ERROR_WANT_X509_LOOKUP)
1355     if (this->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL || SSL_HOOK_OP_TUNNEL == hookOpRequested) {
1356       this->attributes   = HttpProxyPort::TRANSPORT_BLIND_TUNNEL;
1357       sslHandshakeStatus = SSL_HANDSHAKE_ONGOING;
1358       return EVENT_CONT;
1359     } else {
1360       //  Stopping for some other reason, perhaps loading certificate
1361       return SSL_WAIT_FOR_HOOK;
1362     }
1363 #endif
1364 
1365 #if TS_USE_TLS_ASYNC
1366   case SSL_ERROR_WANT_ASYNC:
1367     return SSL_WAIT_FOR_ASYNC;
1368 #endif
1369 
1370   case SSL_ERROR_WANT_ACCEPT:
1371     return EVENT_CONT;
1372 
1373   case SSL_ERROR_SSL: {
1374     SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSLNetVConnection::sslServerHandShakeEvent, SSL_ERROR_SSL errno=%d", errno);
1375     return EVENT_ERROR;
1376   }
1377 
1378   case SSL_ERROR_ZERO_RETURN:
1379     return EVENT_ERROR;
1380   case SSL_ERROR_SYSCALL:
1381     return EVENT_ERROR;
1382   default:
1383     return EVENT_ERROR;
1384   }
1385 }
1386 
1387 int
sslClientHandShakeEvent(int & err)1388 SSLNetVConnection::sslClientHandShakeEvent(int &err)
1389 {
1390   ssl_error_t ssl_error;
1391 
1392   ink_assert(SSLNetVCAccess(ssl) == this);
1393 
1394   // Initialize properly for a client connection
1395   if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) {
1396     sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE;
1397   }
1398 
1399   // Do outbound hook processing here
1400   // Continue on if we are in the invoked state.  The hook has not yet reenabled
1401   if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE) {
1402     return SSL_WAIT_FOR_HOOK;
1403   }
1404 
1405   // Go do the preaccept hooks
1406   if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE) {
1407     SSL_INCREMENT_DYN_STAT(ssl_total_attempts_handshake_count_out_stat);
1408     if (!curHook) {
1409       Debug("ssl", "Initialize outbound connect curHook from NULL");
1410       curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_OUTBOUND_START_HOOK));
1411     } else {
1412       curHook = curHook->next();
1413     }
1414     // If no more hooks, carry on
1415     if (nullptr != curHook) {
1416       sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE;
1417       ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_OUTBOUND_START, this);
1418       return SSL_WAIT_FOR_HOOK;
1419     }
1420   }
1421 
1422   ssl_error = SSLConnect(ssl);
1423   switch (ssl_error) {
1424   case SSL_ERROR_NONE:
1425     if (is_debug_tag_set("ssl")) {
1426       X509 *cert = SSL_get_peer_certificate(ssl);
1427 
1428       Debug("ssl", "SSL client handshake completed successfully");
1429 
1430       if (cert) {
1431         debug_certificate_name("server certificate subject CN is", X509_get_subject_name(cert));
1432         debug_certificate_name("server certificate issuer CN is", X509_get_issuer_name(cert));
1433         X509_free(cert);
1434       }
1435     }
1436 
1437     // if the handshake is complete and write is enabled reschedule the write
1438     if (closed == 0 && write.enabled) {
1439       writeReschedule(nh);
1440     }
1441 
1442     SSL_INCREMENT_DYN_STAT(ssl_total_success_handshake_count_out_stat);
1443 
1444     sslHandshakeStatus = SSL_HANDSHAKE_DONE;
1445     return EVENT_DONE;
1446 
1447   case SSL_ERROR_WANT_WRITE:
1448     Debug("ssl.error", "SSL_ERROR_WANT_WRITE");
1449     return SSL_HANDSHAKE_WANT_WRITE;
1450 
1451   case SSL_ERROR_WANT_READ:
1452     Debug("ssl.error", "SSL_ERROR_WANT_READ");
1453     return SSL_HANDSHAKE_WANT_READ;
1454 #ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB
1455   case SSL_ERROR_WANT_CLIENT_HELLO_CB:
1456     Debug("ssl.error", "SSL_ERROR_WANT_CLIENT_HELLO_CB");
1457     break;
1458 #endif
1459   case SSL_ERROR_WANT_X509_LOOKUP:
1460     Debug("ssl.error", "SSL_ERROR_WANT_X509_LOOKUP");
1461     break;
1462 
1463   case SSL_ERROR_WANT_ACCEPT:
1464     return SSL_HANDSHAKE_WANT_ACCEPT;
1465 
1466   case SSL_ERROR_WANT_CONNECT:
1467     break;
1468 
1469   case SSL_ERROR_ZERO_RETURN:
1470     Debug("ssl.error", "EOS");
1471     return EVENT_ERROR;
1472 
1473   case SSL_ERROR_SYSCALL:
1474     err = errno;
1475     SSL_INCREMENT_DYN_STAT(ssl_error_syscall);
1476     Debug("ssl.error", "syscall");
1477     return EVENT_ERROR;
1478     break;
1479 
1480   case SSL_ERROR_SSL:
1481   default: {
1482     err = (errno) ? errno : -ENET_CONNECT_FAILED;
1483     char buf[512];
1484     unsigned long e = ERR_peek_last_error();
1485     ERR_error_string_n(e, buf, sizeof(buf));
1486     // FIXME -- This triggers a retry on cases of cert validation errors....
1487     SSL_CLR_ERR_INCR_DYN_STAT(this, ssl_error_ssl, "SSL_ERROR_SSL errno=%d", errno);
1488     Debug("ssl.error", "SSL_ERROR_SSL");
1489     if (e) {
1490       if (this->options.sni_servername) {
1491         Debug("ssl.error", "SSL connection failed for '%s': %s", this->options.sni_servername.get(), buf);
1492       } else {
1493         char buff[INET6_ADDRSTRLEN];
1494         ats_ip_ntop(this->get_remote_addr(), buff, INET6_ADDRSTRLEN);
1495         Debug("ssl.error", "SSL connection failed for '%s': %s", buff, buf);
1496       }
1497     }
1498     return EVENT_ERROR;
1499   } break;
1500   }
1501   return EVENT_CONT;
1502 }
1503 
1504 // NextProtocolNegotiation TLS extension callback. The NPN extension
1505 // allows the client to select a preferred protocol, so all we have
1506 // to do here is tell them what out protocol set is.
1507 int
advertise_next_protocol(SSL * ssl,const unsigned char ** out,unsigned int * outlen,void *)1508 SSLNetVConnection::advertise_next_protocol(SSL *ssl, const unsigned char **out, unsigned int *outlen, void * /*arg ATS_UNUSED */)
1509 {
1510   SSLNetVConnection *netvc = SSLNetVCAccess(ssl);
1511 
1512   ink_release_assert(netvc && netvc->ssl == ssl);
1513 
1514   if (netvc->getNPN(out, outlen)) {
1515     // Successful return tells OpenSSL to advertise.
1516     return SSL_TLSEXT_ERR_OK;
1517   }
1518   return SSL_TLSEXT_ERR_NOACK;
1519 }
1520 
1521 // ALPN TLS extension callback. Given the client's set of offered
1522 // protocols, we have to select a protocol to use for this session.
1523 int
select_next_protocol(SSL * ssl,const unsigned char ** out,unsigned char * outlen,const unsigned char * in ATS_UNUSED,unsigned inlen ATS_UNUSED,void *)1524 SSLNetVConnection::select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen,
1525                                         const unsigned char *in ATS_UNUSED, unsigned inlen ATS_UNUSED, void *)
1526 {
1527   SSLNetVConnection *netvc = SSLNetVCAccess(ssl);
1528 
1529   ink_release_assert(netvc && netvc->ssl == ssl);
1530   const unsigned char *npnptr = nullptr;
1531   unsigned int npnsize        = 0;
1532   if (netvc->getNPN(&npnptr, &npnsize)) {
1533     // SSL_select_next_proto chooses the first server-offered protocol that appears in the clients protocol set, ie. the
1534     // server selects the protocol. This is a n^2 search, so it's preferable to keep the protocol set short.
1535     if (SSL_select_next_proto(const_cast<unsigned char **>(out), outlen, npnptr, npnsize, in, inlen) == OPENSSL_NPN_NEGOTIATED) {
1536       Debug("ssl", "selected ALPN protocol %.*s", (int)(*outlen), *out);
1537       return SSL_TLSEXT_ERR_OK;
1538     }
1539   }
1540 
1541   *out    = nullptr;
1542   *outlen = 0;
1543   return SSL_TLSEXT_ERR_NOACK;
1544 }
1545 
1546 void
reenable(NetHandler * nh,int event)1547 SSLNetVConnection::reenable(NetHandler *nh, int event)
1548 {
1549   Debug("ssl", "Handshake reenable from state=%d", sslHandshakeHookState);
1550 
1551   switch (sslHandshakeHookState) {
1552   case HANDSHAKE_HOOKS_PRE_INVOKE:
1553     sslHandshakeHookState = HANDSHAKE_HOOKS_PRE;
1554     break;
1555   case HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE:
1556     sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE;
1557     break;
1558   case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE:
1559     sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO;
1560     break;
1561   case HANDSHAKE_HOOKS_CERT_INVOKE:
1562     sslHandshakeHookState = HANDSHAKE_HOOKS_CERT;
1563     break;
1564   case HANDSHAKE_HOOKS_VERIFY_SERVER:
1565   case HANDSHAKE_HOOKS_CLIENT_CERT:
1566     if (event == TS_EVENT_ERROR) {
1567       sslHandshakeStatus = SSL_HANDSHAKE_ERROR;
1568     }
1569     break;
1570   default:
1571     break;
1572   }
1573 
1574   // Reenabling from the handshake callback
1575   //
1576   // Originally, we would wait for the callback to go again to execute additional
1577   // hooks, but since the callbacks are associated with the context and the context
1578   // can be replaced by the plugin, it didn't seem reasonable to assume that the
1579   // callback would be executed again.  So we walk through the rest of the hooks
1580   // here in the reenable.
1581   if (curHook != nullptr) {
1582     curHook = curHook->next();
1583     Debug("ssl", "iterate from reenable curHook=%p", curHook);
1584   }
1585   if (curHook != nullptr) {
1586     // Invoke the hook and return, wait for next reenable
1587     if (sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_HELLO) {
1588       sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE;
1589       curHook->invoke(TS_EVENT_SSL_CLIENT_HELLO, this);
1590     } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_CLIENT_CERT) {
1591       sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE;
1592       curHook->invoke(TS_EVENT_SSL_VERIFY_CLIENT, this);
1593     } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_CERT) {
1594       sslHandshakeHookState = HANDSHAKE_HOOKS_CERT_INVOKE;
1595       curHook->invoke(TS_EVENT_SSL_CERT, this);
1596     } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_SNI) {
1597       curHook->invoke(TS_EVENT_SSL_SERVERNAME, this);
1598     } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) {
1599       Debug("ssl", "Reenable preaccept");
1600       sslHandshakeHookState = HANDSHAKE_HOOKS_PRE_INVOKE;
1601       ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_START, this);
1602     } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE) {
1603       Debug("ssl", "Reenable outbound connect");
1604       sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE;
1605       ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_OUTBOUND_START, this);
1606     } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_DONE) {
1607       if (this->get_context() == NET_VCONNECTION_OUT) {
1608         ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_OUTBOUND_CLOSE, this);
1609       } else {
1610         ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_CLOSE, this);
1611       }
1612     } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_VERIFY_SERVER) {
1613       Debug("ssl", "ServerVerify");
1614       ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_SSL_VERIFY_SERVER, this);
1615     }
1616     return;
1617   } else {
1618     // Move onto the "next" state
1619     switch (this->sslHandshakeHookState) {
1620     case HANDSHAKE_HOOKS_PRE:
1621     case HANDSHAKE_HOOKS_PRE_INVOKE:
1622       sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO;
1623       break;
1624     case HANDSHAKE_HOOKS_CLIENT_HELLO:
1625     case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE:
1626       sslHandshakeHookState = HANDSHAKE_HOOKS_SNI;
1627       break;
1628     case HANDSHAKE_HOOKS_SNI:
1629       sslHandshakeHookState = HANDSHAKE_HOOKS_CERT;
1630       break;
1631     case HANDSHAKE_HOOKS_CERT:
1632     case HANDSHAKE_HOOKS_CERT_INVOKE:
1633       sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_CERT;
1634       break;
1635     case HANDSHAKE_HOOKS_OUTBOUND_PRE:
1636     case HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE:
1637       this->write.triggered = true;
1638       this->write.enabled   = true;
1639       this->writeReschedule(nh);
1640       sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
1641       break;
1642     case HANDSHAKE_HOOKS_CLIENT_CERT:
1643     case HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE:
1644       sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
1645       break;
1646     case HANDSHAKE_HOOKS_VERIFY_SERVER:
1647       sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
1648       break;
1649     default:
1650       break;
1651     }
1652     Debug("ssl", "iterate from reenable curHook=%p %d", curHook, sslHandshakeHookState);
1653   }
1654 
1655   this->readReschedule(nh);
1656 }
1657 
1658 bool
callHooks(TSEvent eventId)1659 SSLNetVConnection::callHooks(TSEvent eventId)
1660 {
1661   // Only dealing with the SNI/CERT hook so far.
1662   ink_assert(eventId == TS_EVENT_SSL_CLIENT_HELLO || eventId == TS_EVENT_SSL_CERT || eventId == TS_EVENT_SSL_SERVERNAME ||
1663              eventId == TS_EVENT_SSL_VERIFY_SERVER || eventId == TS_EVENT_SSL_VERIFY_CLIENT || eventId == TS_EVENT_VCONN_CLOSE ||
1664              eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE);
1665   Debug("ssl", "sslHandshakeHookState=%d eventID=%d", this->sslHandshakeHookState, eventId);
1666 
1667   // Move state if it is appropriate
1668   switch (this->sslHandshakeHookState) {
1669   case HANDSHAKE_HOOKS_PRE:
1670   case HANDSHAKE_HOOKS_OUTBOUND_PRE:
1671     if (eventId == TS_EVENT_SSL_CLIENT_HELLO) {
1672       this->sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO;
1673     } else if (eventId == TS_EVENT_SSL_SERVERNAME) {
1674       this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI;
1675     } else if (eventId == TS_EVENT_SSL_VERIFY_SERVER) {
1676       this->sslHandshakeHookState = HANDSHAKE_HOOKS_VERIFY_SERVER;
1677     } else if (eventId == TS_EVENT_SSL_CERT) {
1678       this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT;
1679     }
1680     break;
1681   case HANDSHAKE_HOOKS_CLIENT_HELLO:
1682     if (eventId == TS_EVENT_SSL_SERVERNAME) {
1683       this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI;
1684     } else if (eventId == TS_EVENT_SSL_CERT) {
1685       this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT;
1686     } else if (eventId == TS_EVENT_VCONN_CLOSE) {
1687       // Jump to the end
1688       this->sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
1689     }
1690     break;
1691   case HANDSHAKE_HOOKS_SNI:
1692     if (eventId == TS_EVENT_SSL_CERT) {
1693       this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT;
1694     } else if (eventId == TS_EVENT_VCONN_CLOSE) {
1695       // Jump to the end
1696       this->sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
1697     }
1698     break;
1699   default:
1700     break;
1701   }
1702 
1703   // Look for hooks associated with the event
1704   switch (this->sslHandshakeHookState) {
1705   case HANDSHAKE_HOOKS_CLIENT_HELLO:
1706   case HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE:
1707     if (!curHook) {
1708       curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_CLIENT_HELLO_HOOK));
1709     } else {
1710       curHook = curHook->next();
1711     }
1712     if (curHook == nullptr) {
1713       this->sslHandshakeHookState = HANDSHAKE_HOOKS_SNI;
1714     } else {
1715       this->sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE;
1716     }
1717     break;
1718   case HANDSHAKE_HOOKS_VERIFY_SERVER:
1719     // The server verify event addresses ATS to origin handshake
1720     // All the other events are for client to ATS
1721     if (!curHook) {
1722       curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_VERIFY_SERVER_HOOK));
1723     } else {
1724       curHook = curHook->next();
1725     }
1726     break;
1727   case HANDSHAKE_HOOKS_SNI:
1728     if (!curHook) {
1729       curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_SERVERNAME_HOOK));
1730     } else {
1731       curHook = curHook->next();
1732     }
1733     if (!curHook) {
1734       this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT;
1735     }
1736     break;
1737   case HANDSHAKE_HOOKS_CERT:
1738   case HANDSHAKE_HOOKS_CERT_INVOKE:
1739     if (!curHook) {
1740       curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_CERT_HOOK));
1741     } else {
1742       curHook = curHook->next();
1743     }
1744     if (curHook == nullptr) {
1745       this->sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_CERT;
1746     } else {
1747       this->sslHandshakeHookState = HANDSHAKE_HOOKS_CERT_INVOKE;
1748     }
1749     break;
1750   case HANDSHAKE_HOOKS_CLIENT_CERT:
1751   case HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE:
1752     if (!curHook) {
1753       curHook = ssl_hooks->get(TSSslHookInternalID(TS_SSL_VERIFY_CLIENT_HOOK));
1754     } else {
1755       curHook = curHook->next();
1756     }
1757   // fallthrough
1758   case HANDSHAKE_HOOKS_DONE:
1759   case HANDSHAKE_HOOKS_OUTBOUND_PRE:
1760     if (eventId == TS_EVENT_VCONN_CLOSE) {
1761       sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
1762       if (curHook == nullptr) {
1763         curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_CLOSE_HOOK));
1764       } else {
1765         curHook = curHook->next();
1766       }
1767     } else if (eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE) {
1768       sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
1769       if (curHook == nullptr) {
1770         curHook = ssl_hooks->get(TSSslHookInternalID(TS_VCONN_OUTBOUND_CLOSE_HOOK));
1771       } else {
1772         curHook = curHook->next();
1773       }
1774     }
1775     break;
1776   default:
1777     curHook                     = nullptr;
1778     this->sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
1779     return true;
1780   }
1781 
1782   Debug("ssl", "iterated to curHook=%p", curHook);
1783 
1784   bool reenabled = true;
1785 
1786   if (SSL_HOOK_OP_TUNNEL == hookOpRequested) {
1787     this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL;
1788     // Don't mark the handshake as complete yet,
1789     // Will be checking for that flag not being set after
1790     // we get out of this callback, and then will shuffle
1791     // over the buffered handshake packets to the O.S.
1792     // sslHandShakeComplete = 1;
1793     return reenabled;
1794   }
1795 
1796   if (curHook != nullptr) {
1797     WEAK_SCOPED_MUTEX_LOCK(lock, curHook->m_cont->mutex, this_ethread());
1798     curHook->invoke(eventId, this);
1799     reenabled =
1800       (this->sslHandshakeHookState != HANDSHAKE_HOOKS_CERT_INVOKE && this->sslHandshakeHookState != HANDSHAKE_HOOKS_PRE_INVOKE &&
1801        this->sslHandshakeHookState != HANDSHAKE_HOOKS_CLIENT_HELLO_INVOKE);
1802     Debug("ssl", "Called hook on state=%d reenabled=%d", sslHandshakeHookState, reenabled);
1803   }
1804 
1805   return reenabled;
1806 }
1807 
1808 int
populate(Connection & con,Continuation * c,void * arg)1809 SSLNetVConnection::populate(Connection &con, Continuation *c, void *arg)
1810 {
1811   int retval = super::populate(con, c, arg);
1812   if (retval != EVENT_DONE) {
1813     return retval;
1814   }
1815   // Add in the SSL data
1816   this->ssl = static_cast<SSL *>(arg);
1817   // Maybe bring over the stats?
1818 
1819   sslHandshakeStatus = SSL_HANDSHAKE_DONE;
1820   SSLNetVCAttach(this->ssl, this);
1821   TLSSessionResumptionSupport::bind(this->ssl, this);
1822   return EVENT_DONE;
1823 }
1824 
1825 void
increment_ssl_version_metric(int version) const1826 SSLNetVConnection::increment_ssl_version_metric(int version) const
1827 {
1828   switch (version) {
1829   case SSL3_VERSION:
1830     SSL_INCREMENT_DYN_STAT(ssl_total_sslv3);
1831     break;
1832   case TLS1_VERSION:
1833     SSL_INCREMENT_DYN_STAT(ssl_total_tlsv1);
1834     break;
1835   case TLS1_1_VERSION:
1836     SSL_INCREMENT_DYN_STAT(ssl_total_tlsv11);
1837     break;
1838   case TLS1_2_VERSION:
1839     SSL_INCREMENT_DYN_STAT(ssl_total_tlsv12);
1840     break;
1841 #ifdef TLS1_3_VERSION
1842   case TLS1_3_VERSION:
1843     SSL_INCREMENT_DYN_STAT(ssl_total_tlsv13);
1844     break;
1845 #endif
1846   default:
1847     Debug("ssl", "Unrecognized SSL version %d", version);
1848     break;
1849   }
1850 }
1851 
1852 std::string_view
map_tls_protocol_to_tag(const char * proto_string) const1853 SSLNetVConnection::map_tls_protocol_to_tag(const char *proto_string) const
1854 {
1855   std::string_view retval{"tls/?.?"sv}; // return this if the protocol lookup doesn't work.
1856 
1857   if (proto_string) {
1858     // openSSL guarantees the case of the protocol string.
1859     if (proto_string[0] == 'T' && proto_string[1] == 'L' && proto_string[2] == 'S' && proto_string[3] == 'v' &&
1860         proto_string[4] == '1') {
1861       if (proto_string[5] == 0) {
1862         retval = IP_PROTO_TAG_TLS_1_0;
1863       } else if (proto_string[5] == '.' && proto_string[7] == 0) {
1864         switch (proto_string[6]) {
1865         case '1':
1866           retval = IP_PROTO_TAG_TLS_1_1;
1867           break;
1868         case '2':
1869           retval = IP_PROTO_TAG_TLS_1_2;
1870           break;
1871         case '3':
1872           retval = IP_PROTO_TAG_TLS_1_3;
1873           break;
1874         default:
1875           break;
1876         }
1877       }
1878     }
1879   }
1880   return retval;
1881 }
1882 
1883 int
populate_protocol(std::string_view * results,int n) const1884 SSLNetVConnection::populate_protocol(std::string_view *results, int n) const
1885 {
1886   int retval = 0;
1887   if (n > retval) {
1888     results[retval] = map_tls_protocol_to_tag(getSSLProtocol());
1889     if (!results[retval].empty()) {
1890       ++retval;
1891     }
1892     if (n > retval) {
1893       retval += super::populate_protocol(results + retval, n - retval);
1894     }
1895   }
1896   return retval;
1897 }
1898 
1899 const char *
protocol_contains(std::string_view prefix) const1900 SSLNetVConnection::protocol_contains(std::string_view prefix) const
1901 {
1902   const char *retval   = nullptr;
1903   std::string_view tag = map_tls_protocol_to_tag(getSSLProtocol());
1904   if (prefix.size() <= tag.size() && strncmp(tag.data(), prefix.data(), prefix.size()) == 0) {
1905     retval = tag.data();
1906   } else {
1907     retval = super::protocol_contains(prefix);
1908   }
1909   return retval;
1910 }
1911 
1912 void
set_server_name(std::string_view name)1913 SSLNetVConnection::set_server_name(std::string_view name)
1914 {
1915   if (name.size()) {
1916     char *n = new char[name.size() + 1];
1917     std::memcpy(n, name.data(), name.size());
1918     n[name.size()] = '\0';
1919     _serverName.reset(n);
1920   }
1921 }
1922