1 /** @file
2 
3 This is an origin server / intercept plugin, which implements flexible health checks.
4 
5 @section license
6 
7 Licensed under the Apache License, Version 2.0 (the "License");
8 you may not use this file except in compliance with the License.
9 You may obtain a copy of the License at
10 
11 http://www.apache.org/licenses/LICENSE-2.0
12 
13 Unless required by applicable law or agreed to in writing, software
14 distributed under the License is distributed on an "AS IS" BASIS,
15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 See the License for the specific language governing permissions and
17 limitations under the License.
18  */
19 
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <ctype.h>
23 #include <limits.h>
24 #include <string.h>
25 #include <sys/types.h>
26 #include <sys/stat.h>
27 #include <sys/time.h>
28 #include <unistd.h>
29 #include <inttypes.h>
30 
31 /* ToDo: Linux specific */
32 #include <sys/inotify.h>
33 #include <libgen.h>
34 
35 #include "ts/ts.h"
36 #include "tscore/ink_platform.h"
37 #include "tscore/ink_defs.h"
38 
39 static const char PLUGIN_NAME[] = "healthchecks";
40 static const char SEPARATORS[]  = " \t\n";
41 
42 #define MAX_PATH_LEN 4096
43 #define MAX_BODY_LEN 16384
44 #define FREELIST_TIMEOUT 300
45 
46 static inline void *
ink_atomic_swap_ptr(void * mem,void * value)47 ink_atomic_swap_ptr(void *mem, void *value)
48 {
49   return __sync_lock_test_and_set((void **)mem, value);
50 }
51 
52 /* Directories that we are watching for inotify IN_CREATE events. */
53 typedef struct HCDirEntry_t {
54   char dname[MAX_PATH_LEN];   /* Directory name */
55   int wd;                     /* Watch descriptor */
56   struct HCDirEntry_t *_next; /* Linked list */
57 } HCDirEntry;
58 
59 /* Information about a status file. This is never modified (only replaced, see HCFileInfo_t) */
60 typedef struct HCFileData_t {
61   int exists;                 /* Does this file exist */
62   char body[MAX_BODY_LEN];    /* Body from fname. NULL means file is missing */
63   int b_len;                  /* Length of data */
64   time_t remove;              /* Used for deciding when the old object can be permanently removed */
65   struct HCFileData_t *_next; /* Only used when these guys end up on the freelist */
66 } HCFileData;
67 
68 /* The only thing that should change in this struct is data, atomically swapping ptrs */
69 typedef struct HCFileInfo_t {
70   char fname[MAX_PATH_LEN];   /* Filename */
71   char *basename;             /* The "basename" of the file */
72   char path[PATH_NAME_MAX];   /* URL path for this HC */
73   int p_len;                  /* Length of path */
74   const char *ok;             /* Header for an OK result */
75   int o_len;                  /* Length of OK header */
76   const char *miss;           /* Header for miss results */
77   int m_len;                  /* Length of miss header */
78   HCFileData *data;           /* Holds the current data for this health check file */
79   int wd;                     /* Watch descriptor */
80   HCDirEntry *dir;            /* Reference to the directory this file resides in */
81   struct HCFileInfo_t *_next; /* Linked list */
82 } HCFileInfo;
83 
84 /* Global configuration */
85 HCFileInfo *g_config;
86 
87 /* State used for the intercept plugin. ToDo: Can this be improved ? */
88 typedef struct HCState_t {
89   TSVConn net_vc;
90   TSVIO read_vio;
91   TSVIO write_vio;
92 
93   TSIOBuffer req_buffer;
94   TSIOBuffer resp_buffer;
95   TSIOBufferReader resp_reader;
96 
97   int output_bytes;
98 
99   /* We actually need both here, so that our lock free switches works safely */
100   HCFileInfo *info;
101   HCFileData *data;
102 } HCState;
103 
104 /* Read / check the status files */
105 static void
reload_status_file(HCFileInfo * info,HCFileData * data)106 reload_status_file(HCFileInfo *info, HCFileData *data)
107 {
108   FILE *fd;
109 
110   memset(data, 0, sizeof(HCFileData));
111   if (NULL != (fd = fopen(info->fname, "r"))) {
112     data->exists = 1;
113     do {
114       data->b_len = fread(data->body, 1, MAX_BODY_LEN, fd);
115     } while (!feof(fd)); /*  Only save the last 16KB of the file ... */
116     fclose(fd);
117   }
118 }
119 
120 /* Find a HCDirEntry from the linked list */
121 static HCDirEntry *
find_direntry(const char * dname,HCDirEntry * dir)122 find_direntry(const char *dname, HCDirEntry *dir)
123 {
124   while (dir) {
125     if (!strncmp(dname, dir->dname, MAX_PATH_LEN)) {
126       return dir;
127     }
128     dir = dir->_next;
129   }
130   return NULL;
131 }
132 
133 /* Setup up watchers, directory as well as initial files */
134 static HCDirEntry *
setup_watchers(int fd)135 setup_watchers(int fd)
136 {
137   HCFileInfo *conf     = g_config;
138   HCDirEntry *head_dir = NULL, *last_dir = NULL, *dir;
139   char fname[MAX_PATH_LEN];
140 
141   while (conf) {
142     conf->wd = inotify_add_watch(fd, conf->fname, IN_DELETE_SELF | IN_CLOSE_WRITE | IN_ATTRIB);
143     TSDebug(PLUGIN_NAME, "Setting up a watcher for %s", conf->fname);
144     strncpy(fname, conf->fname, MAX_PATH_LEN);
145     char *dname = dirname(fname);
146     /* Make sure to only watch each directory once */
147     if (!(dir = find_direntry(dname, head_dir))) {
148       TSDebug(PLUGIN_NAME, "Setting up a watcher for directory %s", dname);
149       dir = TSmalloc(sizeof(HCDirEntry));
150       memset(dir, 0, sizeof(HCDirEntry));
151       strncpy(dir->dname, dname, MAX_PATH_LEN - 1);
152       dir->wd = inotify_add_watch(fd, dname, IN_CREATE | IN_MOVED_FROM | IN_MOVED_TO | IN_ATTRIB);
153       if (!head_dir) {
154         head_dir = dir;
155       } else {
156         last_dir->_next = dir;
157       }
158       last_dir = dir;
159     }
160     conf->dir = dir;
161     conf      = conf->_next;
162   }
163 
164   return head_dir;
165 }
166 
167 /* Separate thread to monitor status files for reload */
168 #define INOTIFY_BUFLEN (1024 * sizeof(struct inotify_event))
169 
170 static void *
hc_thread(void * data ATS_UNUSED)171 hc_thread(void *data ATS_UNUSED)
172 {
173   int fd              = inotify_init();
174   HCFileData *fl_head = NULL;
175   char buffer[INOTIFY_BUFLEN];
176   struct timeval last_free, now;
177 
178   gettimeofday(&last_free, NULL);
179 
180   /* Setup watchers for the directories, these are a one time setup */
181   setup_watchers(fd); // This is a leak, but since we enter an infinite loop this is ok?
182 
183   while (1) {
184     HCFileData *fdata = fl_head, *fdata_prev = NULL;
185 
186     gettimeofday(&now, NULL);
187     /* Read the inotify events, blocking until we get something */
188     int len = read(fd, buffer, INOTIFY_BUFLEN);
189 
190     /* The fl_head is a linked list of previously released data entries. They
191        are ordered "by time", so once we find one that is scheduled for deletion,
192        we can also delete all entries after it in the linked list. */
193     while (fdata) {
194       if (now.tv_sec > fdata->remove) {
195         /* Now drop off the "tail" from the freelist */
196         if (fdata_prev) {
197           fdata_prev->_next = NULL;
198         } else {
199           fl_head = NULL;
200         }
201 
202         /* free() everything in the "tail" */
203         do {
204           HCFileData *next = fdata->_next;
205 
206           TSDebug(PLUGIN_NAME, "Cleaning up entry from freelist");
207           TSfree(fdata);
208           fdata = next;
209         } while (fdata);
210         break; /* Stop the loop, there's nothing else left to examine */
211       }
212       fdata_prev = fdata;
213       fdata      = fdata->_next;
214     }
215 
216     if (len >= 0) {
217       int i = 0;
218 
219       while (i < len) {
220         struct inotify_event *event = (struct inotify_event *)&buffer[i];
221         HCFileInfo *finfo           = g_config;
222 
223         while (finfo && !((event->wd == finfo->wd) ||
224                           ((event->wd == finfo->dir->wd) && !strncmp(event->name, finfo->basename, event->len)))) {
225           finfo = finfo->_next;
226         }
227         if (finfo) {
228           HCFileData *new_data = TSmalloc(sizeof(HCFileData));
229           HCFileData *old_data;
230 
231           if (event->mask & (IN_CLOSE_WRITE | IN_ATTRIB)) {
232             TSDebug(PLUGIN_NAME, "Modify file event (%d) on %s", event->mask, finfo->fname);
233           } else if (event->mask & (IN_CREATE | IN_MOVED_TO)) {
234             TSDebug(PLUGIN_NAME, "Create file event (%d) on %s", event->mask, finfo->fname);
235             finfo->wd = inotify_add_watch(fd, finfo->fname, IN_DELETE_SELF | IN_CLOSE_WRITE | IN_ATTRIB);
236           } else if (event->mask & (IN_DELETE_SELF | IN_MOVED_FROM)) {
237             TSDebug(PLUGIN_NAME, "Delete file event (%d) on %s", event->mask, finfo->fname);
238             finfo->wd = inotify_rm_watch(fd, finfo->wd);
239           }
240           /* Load the new data and then swap this atomically */
241           memset(new_data, 0, sizeof(HCFileData));
242           reload_status_file(finfo, new_data);
243           TSDebug(PLUGIN_NAME, "Reloaded %s, len == %d, exists == %d", finfo->fname, new_data->b_len, new_data->exists);
244           old_data = ink_atomic_swap_ptr(&(finfo->data), new_data);
245 
246           /* Add the old data to the head of the freelist */
247           old_data->remove = now.tv_sec + FREELIST_TIMEOUT;
248           old_data->_next  = fl_head;
249           fl_head          = old_data;
250         }
251         /* coverity[ -tainted_data_return] */
252         i += sizeof(struct inotify_event) + event->len;
253       }
254     }
255   }
256 
257   return NULL; /* Yeah, that never happens */
258 }
259 
260 /* Config file parsing */
261 static const char HEADER_TEMPLATE[] = "HTTP/1.1 %d %s\r\nContent-Type: %s\r\nCache-Control: no-cache\r\n";
262 
263 static char *
gen_header(char * status_str,char * mime,int * header_len)264 gen_header(char *status_str, char *mime, int *header_len)
265 {
266   TSHttpStatus status;
267   char *buf = NULL;
268 
269   status = atoi(status_str);
270   if (status > TS_HTTP_STATUS_NONE && status < (TSHttpStatus)999) {
271     const char *status_reason;
272     int len = sizeof(HEADER_TEMPLATE) + 3 + 1;
273 
274     status_reason = TSHttpHdrReasonLookup(status);
275     len += strlen(status_reason);
276     len += strlen(mime);
277     buf         = TSmalloc(len);
278     *header_len = snprintf(buf, len, HEADER_TEMPLATE, status, status_reason, mime);
279   } else {
280     *header_len = 0;
281   }
282 
283   return buf;
284 }
285 
286 static HCFileInfo *
parse_configs(const char * fname)287 parse_configs(const char *fname)
288 {
289   FILE *fd;
290   char buf[2 * 1024];
291   HCFileInfo *head_finfo = NULL, *finfo = NULL, *prev_finfo = NULL;
292 
293   if (!fname) {
294     return NULL;
295   }
296 
297   if ('/' == *fname) {
298     fd = fopen(fname, "r");
299   } else {
300     char filename[PATH_MAX + 1];
301 
302     snprintf(filename, sizeof(filename), "%s/%s", TSConfigDirGet(), fname);
303     fd = fopen(filename, "r");
304   }
305 
306   if (NULL == fd) {
307     TSError("%s: Could not open config file", PLUGIN_NAME);
308     return NULL;
309   }
310 
311   while (!feof(fd)) {
312     char *str, *save;
313     char *ok = NULL, *miss = NULL, *mime = NULL;
314 
315     finfo = TSmalloc(sizeof(HCFileInfo));
316     memset(finfo, 0, sizeof(HCFileInfo));
317 
318     if (fgets(buf, sizeof(buf) - 1, fd)) {
319       str       = strtok_r(buf, SEPARATORS, &save);
320       int state = 0;
321       while (NULL != str) {
322         if (strlen(str) > 0) {
323           switch (state) {
324           case 0:
325             if ('/' == *str) {
326               ++str;
327             }
328             strncpy(finfo->path, str, PATH_NAME_MAX - 1);
329             finfo->p_len = strlen(finfo->path);
330             break;
331           case 1:
332             strncpy(finfo->fname, str, MAX_PATH_LEN - 1);
333             finfo->basename = strrchr(finfo->fname, '/');
334             if (finfo->basename) {
335               ++(finfo->basename);
336             }
337             break;
338           case 2:
339             mime = str;
340             break;
341           case 3:
342             ok = str;
343             break;
344           case 4:
345             miss = str;
346             break;
347           }
348           ++state;
349         }
350         str = strtok_r(NULL, SEPARATORS, &save);
351       }
352 
353       /* Fill in the info if everything was ok */
354       if (state > 4) {
355         TSDebug(PLUGIN_NAME, "Parsed: %s %s %s %s %s", finfo->path, finfo->fname, mime, ok, miss);
356         finfo->ok   = gen_header(ok, mime, &finfo->o_len);
357         finfo->miss = gen_header(miss, mime, &finfo->m_len);
358         finfo->data = TSmalloc(sizeof(HCFileData));
359         memset(finfo->data, 0, sizeof(HCFileData));
360         reload_status_file(finfo, finfo->data);
361 
362         /* Add it the linked list */
363         TSDebug(PLUGIN_NAME, "Adding path=%s to linked list", finfo->path);
364         if (NULL == head_finfo) {
365           head_finfo = finfo;
366         } else {
367           prev_finfo->_next = finfo;
368         }
369         prev_finfo = finfo;
370       } else {
371         TSfree(finfo);
372       }
373     }
374   }
375   fclose(fd);
376 
377   return head_finfo;
378 }
379 
380 /* Cleanup after intercept has completed */
381 static void
cleanup(TSCont contp,HCState * my_state)382 cleanup(TSCont contp, HCState *my_state)
383 {
384   if (my_state->req_buffer) {
385     TSIOBufferDestroy(my_state->req_buffer);
386     my_state->req_buffer = NULL;
387   }
388 
389   if (my_state->resp_buffer) {
390     TSIOBufferDestroy(my_state->resp_buffer);
391     my_state->resp_buffer = NULL;
392   }
393 
394   TSVConnClose(my_state->net_vc);
395   TSfree(my_state);
396   TSContDestroy(contp);
397 }
398 
399 /* Add data to the output */
400 inline static int
add_data_to_resp(const char * buf,int len,HCState * my_state)401 add_data_to_resp(const char *buf, int len, HCState *my_state)
402 {
403   TSIOBufferWrite(my_state->resp_buffer, buf, len);
404   return len;
405 }
406 
407 /* Process a read event from the SM */
408 static void
hc_process_read(TSCont contp,TSEvent event,HCState * my_state)409 hc_process_read(TSCont contp, TSEvent event, HCState *my_state)
410 {
411   if (event == TS_EVENT_VCONN_READ_READY) {
412     if (my_state->data->exists) {
413       TSDebug(PLUGIN_NAME, "Setting OK response header");
414       my_state->output_bytes = add_data_to_resp(my_state->info->ok, my_state->info->o_len, my_state);
415     } else {
416       TSDebug(PLUGIN_NAME, "Setting MISS response header");
417       my_state->output_bytes = add_data_to_resp(my_state->info->miss, my_state->info->m_len, my_state);
418     }
419     TSVConnShutdown(my_state->net_vc, 1, 0);
420     my_state->write_vio = TSVConnWrite(my_state->net_vc, contp, my_state->resp_reader, INT64_MAX);
421   } else if (event == TS_EVENT_ERROR) {
422     TSError("[healthchecks] hc_process_read: Received TS_EVENT_ERROR");
423   } else if (event == TS_EVENT_VCONN_EOS) {
424     /* client may end the connection, simply return */
425     return;
426   } else if (event == TS_EVENT_NET_ACCEPT_FAILED) {
427     TSError("[healthchecks] hc_process_read: Received TS_EVENT_NET_ACCEPT_FAILED");
428   } else {
429     TSReleaseAssert(!"Unexpected Event");
430   }
431 }
432 
433 /* Process a write event from the SM */
434 static void
hc_process_write(TSCont contp,TSEvent event,HCState * my_state)435 hc_process_write(TSCont contp, TSEvent event, HCState *my_state)
436 {
437   if (event == TS_EVENT_VCONN_WRITE_READY) {
438     char buf[48];
439     int len;
440 
441     len = snprintf(buf, sizeof(buf), "Content-Length: %d\r\n\r\n", my_state->data->b_len);
442     my_state->output_bytes += add_data_to_resp(buf, len, my_state);
443     if (my_state->data->b_len > 0) {
444       my_state->output_bytes += add_data_to_resp(my_state->data->body, my_state->data->b_len, my_state);
445     } else {
446       my_state->output_bytes += add_data_to_resp("\r\n", 2, my_state);
447     }
448     TSVIONBytesSet(my_state->write_vio, my_state->output_bytes);
449     TSVIOReenable(my_state->write_vio);
450   } else if (event == TS_EVENT_VCONN_WRITE_COMPLETE) {
451     cleanup(contp, my_state);
452   } else if (event == TS_EVENT_ERROR) {
453     TSError("[healthchecks] hc_process_write: Received TS_EVENT_ERROR");
454   } else {
455     TSReleaseAssert(!"Unexpected Event");
456   }
457 }
458 
459 /* Process the accept event from the SM */
460 static void
hc_process_accept(TSCont contp,HCState * my_state)461 hc_process_accept(TSCont contp, HCState *my_state)
462 {
463   my_state->req_buffer  = TSIOBufferCreate();
464   my_state->resp_buffer = TSIOBufferCreate();
465   my_state->resp_reader = TSIOBufferReaderAlloc(my_state->resp_buffer);
466   my_state->read_vio    = TSVConnRead(my_state->net_vc, contp, my_state->req_buffer, INT64_MAX);
467 }
468 
469 /* Implement the server intercept */
470 static int
hc_intercept(TSCont contp,TSEvent event,void * edata)471 hc_intercept(TSCont contp, TSEvent event, void *edata)
472 {
473   HCState *my_state = TSContDataGet(contp);
474 
475   if (event == TS_EVENT_NET_ACCEPT) {
476     my_state->net_vc = (TSVConn)edata;
477     hc_process_accept(contp, my_state);
478   } else if (edata == my_state->read_vio) { /* All read events */
479     hc_process_read(contp, event, my_state);
480   } else if (edata == my_state->write_vio) { /* All write events */
481     hc_process_write(contp, event, my_state);
482   } else {
483     TSReleaseAssert(!"Unexpected Event");
484   }
485 
486   return 0;
487 }
488 
489 /* Read-request header continuation, used to kick off the server intercept if necessary */
490 static int
health_check_origin(TSCont contp ATS_UNUSED,TSEvent event ATS_UNUSED,void * edata)491 health_check_origin(TSCont contp ATS_UNUSED, TSEvent event ATS_UNUSED, void *edata)
492 {
493   TSMBuffer reqp;
494   TSMLoc hdr_loc = NULL, url_loc = NULL;
495   TSCont icontp;
496   HCState *my_state;
497   TSHttpTxn txnp   = (TSHttpTxn)edata;
498   HCFileInfo *info = g_config;
499 
500   if ((TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &reqp, &hdr_loc)) && (TS_SUCCESS == TSHttpHdrUrlGet(reqp, hdr_loc, &url_loc))) {
501     int path_len     = 0;
502     const char *path = TSUrlPathGet(reqp, url_loc, &path_len);
503 
504     /* Short circuit the / path, common case, and we won't allow healthchecks on / */
505     if (!path || !path_len) {
506       goto cleanup;
507     }
508 
509     while (info) {
510       if (info->p_len == path_len && !memcmp(info->path, path, path_len)) {
511         TSDebug(PLUGIN_NAME, "Found match for /%.*s", path_len, path);
512         break;
513       }
514       info = info->_next;
515     }
516 
517     if (!info) {
518       goto cleanup;
519     }
520 
521     TSSkipRemappingSet(txnp, 1); /* not strictly necessary, but speed is everything these days */
522 
523     /* This is us -- register our intercept */
524     icontp   = TSContCreate(hc_intercept, TSMutexCreate());
525     my_state = (HCState *)TSmalloc(sizeof(*my_state));
526     memset(my_state, 0, sizeof(*my_state));
527     my_state->info = info;
528     my_state->data = info->data;
529     TSContDataSet(icontp, my_state);
530     TSHttpTxnIntercept(icontp, txnp);
531   }
532 
533 cleanup:
534   if (url_loc) {
535     TSHandleMLocRelease(reqp, hdr_loc, url_loc);
536   }
537   if (hdr_loc) {
538     TSHandleMLocRelease(reqp, TS_NULL_MLOC, hdr_loc);
539   }
540 
541   TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
542 
543   return 0;
544 }
545 
546 /* Initialize the plugin / global continuation hook */
547 void
TSPluginInit(int argc,const char * argv[])548 TSPluginInit(int argc, const char *argv[])
549 {
550   TSPluginRegistrationInfo info;
551 
552   if (2 != argc) {
553     TSError("[healthchecks] Must specify a configuration file");
554     return;
555   }
556 
557   info.plugin_name   = "health_checks";
558   info.vendor_name   = "Apache Software Foundation";
559   info.support_email = "dev@trafficserver.apache.org";
560 
561   if (TS_SUCCESS != TSPluginRegister(&info)) {
562     TSError("[healthchecks] Plugin registration failed");
563     return;
564   }
565 
566   /* This will update the global configuration file, and is not reloaded at run time */
567   /* ToDo: Support reloading with traffic_ctl config reload ? */
568   if (NULL == (g_config = parse_configs(argv[1]))) {
569     TSError("[healthchecks] Unable to read / parse %s config file", argv[1]);
570     return;
571   }
572 
573   /* Setup the background thread */
574   if (!TSThreadCreate(hc_thread, NULL)) {
575     TSError("[healthchecks] Failure in thread creation");
576     return;
577   }
578 
579   /* Create a continuation with a mutex as there is a shared global structure
580      containing the headers to add */
581   TSDebug(PLUGIN_NAME, "Started %s plugin", PLUGIN_NAME);
582   TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(health_check_origin, NULL));
583 }
584