1 /** @file
2 
3   Contains main function definition for logcat
4 
5   @section license License
6 
7   Licensed to the Apache Software Foundation (ASF) under one
8   or more contributor license agreements.  See the NOTICE file
9   distributed with this work for additional information
10   regarding copyright ownership.  The ASF licenses this file
11   to you under the Apache License, Version 2.0 (the
12   "License"); you may not use this file except in compliance
13   with the License.  You may obtain a copy of the License at
14 
15       http://www.apache.org/licenses/LICENSE-2.0
16 
17   Unless required by applicable law or agreed to in writing, software
18   distributed under the License is distributed on an "AS IS" BASIS,
19   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20   See the License for the specific language governing permissions and
21   limitations under the License.
22  */
23 
24 #include "tscore/ink_platform.h"
25 #include "tscore/ink_args.h"
26 #include "tscore/I_Layout.h"
27 #include "tscore/runroot.h"
28 
29 #define PROGRAM_NAME "traffic_logcat"
30 #define MAX_LOGBUFFER_SIZE 65536
31 
32 #include <poll.h>
33 
34 #include "LogStandalone.cc"
35 
36 #include "LogAccess.h"
37 #include "LogField.h"
38 #include "LogFilter.h"
39 #include "LogFormat.h"
40 #include "LogFile.h"
41 #include "LogObject.h"
42 #include "LogConfig.h"
43 #include "LogBuffer.h"
44 #include "LogUtils.h"
45 #include "Log.h"
46 
47 // logcat-specific command-line flags
48 static int squid_flag              = 0;
49 static int follow_flag             = 0;
50 static int clf_flag                = 0;
51 static int elf_flag                = 0;
52 static int elf2_flag               = 0;
53 static int auto_filenames          = 0;
54 static int overwrite_existing_file = 0;
55 static char output_file[1024];
56 int auto_clear_cache_flag = 0;
57 
58 static const ArgumentDescription argument_descriptions[] = {
59 
60   {"output_file", 'o', "Specify output file", "S1023", &output_file, NULL, NULL},
61   {"auto_filenames", 'a', "Automatically generate output names", "T", &auto_filenames, NULL, NULL},
62   {"follow", 'f', "Follow the log file as it grows", "T", &follow_flag, NULL, NULL},
63   {"clf", 'C', "Convert to Common Logging Format", "T", &clf_flag, NULL, NULL},
64   {"elf", 'E', "Convert to Extended Logging Format", "T", &elf_flag, NULL, NULL},
65   {"squid", 'S', "Convert to Squid Logging Format", "T", &squid_flag, NULL, NULL},
66   {"debug_tags", 'T', "Colon-Separated Debug Tags", "S1023", error_tags, NULL, NULL},
67   {"overwrite_output", 'w', "Overwrite existing output file(s)", "T", &overwrite_existing_file, NULL, NULL},
68   {"elf2", '2', "Convert to Extended2 Logging Format", "T", &elf2_flag, NULL, NULL},
69   HELP_ARGUMENT_DESCRIPTION(),
70   VERSION_ARGUMENT_DESCRIPTION(),
71   RUNROOT_ARGUMENT_DESCRIPTION()};
72 
73 /*
74  * Gets the inode number of a given file
75  *
76  * @param filename name of the file
77  * @returns -1 on failure, otherwise inode number
78  */
79 static ino_t
get_inode_num(const char * filename)80 get_inode_num(const char *filename)
81 {
82   struct stat sb;
83 
84   if (stat(filename, &sb) != 0) {
85     perror("stat");
86     return -1;
87   }
88 
89   return sb.st_ino;
90 }
91 
92 /*
93  * Checks if a log file has been rotated, and if so, opens the rotated file
94  * and returns the new file descriptor
95  *
96  * @param input_file name of log file we want to follow
97  * @param old_inode_num the most recently known inode number of `input_name`
98  * @returns -1 on failure, 0 on noop, otherwise the open fd of rotated file
99  */
100 static int
follow_rotate(const char * input_file,ino_t old_inode_num)101 follow_rotate(const char *input_file, ino_t old_inode_num)
102 {
103   // check if file has been rotated
104   if (get_inode_num(input_file) != old_inode_num) {
105     int new_fd = open(input_file, O_RDONLY);
106     if (new_fd < 0) {
107       fprintf(stderr, "Error while trying to follow rotated input file %s: %s\n", input_file, strerror(errno));
108       return -1;
109     }
110 
111     return new_fd;
112   } else { // file has not been rotated
113     return 0;
114   }
115 }
116 
117 static int
process_file(int in_fd,int out_fd)118 process_file(int in_fd, int out_fd)
119 {
120   char buffer[MAX_LOGBUFFER_SIZE];
121   int nread, buffer_bytes;
122   unsigned bytes = 0;
123 
124   while (true) {
125     // read the next buffer from file descriptor
126     //
127     Debug("logcat", "Reading buffer ...");
128     memset(buffer, 0, sizeof(buffer));
129 
130     // read the first 8 bytes of the header, which will give us the
131     // cookie and the version number.
132     //
133     unsigned first_read_size = sizeof(uint32_t) + sizeof(uint32_t);
134     unsigned header_size     = sizeof(LogBufferHeader);
135     LogBufferHeader *header  = (LogBufferHeader *)&buffer[0];
136 
137     nread = read(in_fd, buffer, first_read_size);
138     if (!nread || nread == EOF) {
139       return 0;
140     }
141 
142     // ensure that this is a valid logbuffer header
143     //
144     if (header->cookie != LOG_SEGMENT_COOKIE) {
145       fprintf(stderr, "Bad LogBuffer!\n");
146       return 1;
147     }
148     // read the rest of the header
149     //
150     unsigned second_read_size = header_size - first_read_size;
151 
152     nread = read(in_fd, &buffer[first_read_size], second_read_size);
153     if (!nread || nread == EOF) {
154       if (follow_flag) {
155         return 0;
156       }
157 
158       fprintf(stderr, "Bad LogBufferHeader read!\n");
159       return 1;
160     }
161     // read the rest of the buffer
162     //
163     uint32_t byte_count = header->byte_count;
164 
165     if (byte_count > sizeof(buffer)) {
166       fprintf(stderr, "Buffer too large!\n");
167       return 1;
168     }
169     buffer_bytes = byte_count - header_size;
170     if (buffer_bytes == 0) {
171       return 0;
172     }
173     if (buffer_bytes < 0) {
174       fprintf(stderr, "No buffer body!\n");
175       return 1;
176     }
177     // Read the next full buffer (allowing for "partial" reads)
178     nread = 0;
179     while (nread < buffer_bytes) {
180       int rc = read(in_fd, &buffer[header_size] + nread, buffer_bytes - nread);
181 
182       if ((rc == EOF) && (!follow_flag)) {
183         fprintf(stderr, "Bad LogBuffer read!\n");
184         return 1;
185       }
186 
187       if (rc > 0) {
188         nread += rc;
189       }
190     }
191 
192     if (nread > buffer_bytes) {
193       fprintf(stderr, "Read too many bytes!\n");
194       return 1;
195     }
196     // see if there is an alternate format request from the command
197     // line
198     //
199     const char *alt_format = nullptr;
200     // convert the buffer to ascii entries and place onto stdout
201     //
202     if (header->fmt_fieldlist()) {
203       bytes += LogFile::write_ascii_logbuffer(header, out_fd, ".", alt_format);
204     } else {
205       // TODO investigate why this buffer goes wonky
206     }
207   }
208 }
209 
210 static int
open_output_file(char * output_file)211 open_output_file(char *output_file)
212 {
213   int file_desc = 0;
214 
215   if (!overwrite_existing_file) {
216     if (access(output_file, F_OK)) {
217       if (errno != ENOENT) {
218         fprintf(stderr, "Error accessing output file %s: ", output_file);
219         perror(nullptr);
220         file_desc = -1;
221       }
222     } else {
223       fprintf(stderr,
224               "Error, output file %s already exists.\n"
225               "Select a different filename or use the -w flag\n",
226               output_file);
227       file_desc = -1;
228     }
229   }
230 
231   if (file_desc == 0) {
232     file_desc = open(output_file, O_WRONLY | O_TRUNC | O_CREAT, 0640);
233 
234     if (file_desc < 0) {
235       fprintf(stderr, "Error while opening output file %s: ", output_file);
236       perror(nullptr);
237     }
238   }
239 
240   return file_desc;
241 }
242 
243 /*-------------------------------------------------------------------------
244   main
245   -------------------------------------------------------------------------*/
246 
247 int
main(int,const char * argv[])248 main(int /* argc ATS_UNUSED */, const char *argv[])
249 {
250   enum {
251     NO_ERROR              = 0,
252     CMD_LINE_OPTION_ERROR = 1,
253     DATA_PROCESSING_ERROR = 2,
254   };
255 
256   // build the application information structure
257   //
258   appVersionInfo.setup(PACKAGE_NAME, PROGRAM_NAME, PACKAGE_VERSION, __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, "");
259 
260   runroot_handler(argv);
261   // Before accessing file system initialize Layout engine
262   Layout::create();
263   // process command-line arguments
264   //
265   output_file[0] = 0;
266   process_args(&appVersionInfo, argument_descriptions, countof(argument_descriptions), argv);
267 
268   // check that only one of the -o and -a options was specified
269   //
270   if (output_file[0] != 0 && auto_filenames) {
271     fprintf(stderr, "Error: specify only one of -o <file> and -a\n");
272     ::exit(CMD_LINE_OPTION_ERROR);
273   }
274   // initialize this application for standalone logging operation
275   //
276   init_log_standalone_basic(PROGRAM_NAME);
277 
278   Log::init(Log::NO_REMOTE_MANAGEMENT | Log::LOGCAT);
279 
280   // setup output file
281   //
282   int out_fd = STDOUT_FILENO;
283   if (output_file[0] != 0) {
284     out_fd = open_output_file(output_file);
285 
286     if (out_fd < 0) {
287       ::exit(DATA_PROCESSING_ERROR);
288     }
289   } else if (!auto_filenames) {
290     out_fd = STDOUT_FILENO;
291   }
292   // process file arguments
293   //
294   int error = NO_ERROR;
295 
296   if (n_file_arguments) {
297     int bin_ext_len   = strlen(LOG_FILE_BINARY_OBJECT_FILENAME_EXTENSION);
298     int ascii_ext_len = strlen(LOG_FILE_ASCII_OBJECT_FILENAME_EXTENSION);
299 
300     for (unsigned i = 0; i < n_file_arguments; ++i) {
301       int in_fd = open(file_arguments[i], O_RDONLY);
302       if (in_fd < 0) {
303         fprintf(stderr, "Error opening input file %s: ", file_arguments[i]);
304         perror(nullptr);
305         error = DATA_PROCESSING_ERROR;
306       } else {
307 #if HAVE_POSIX_FADVISE
308         // If we don't plan on following the log file, we should let the kernel know
309         // that we plan on reading the entire file so the kernel can do
310         // some fancy optimizations.
311         if (!follow_flag) {
312           posix_fadvise(in_fd, 0, 0, POSIX_FADV_WILLNEED);
313         }
314 
315         // We're always reading the file sequentially so this will always help
316         posix_fadvise(in_fd, 0, 0, POSIX_FADV_SEQUENTIAL);
317 #endif
318         if (auto_filenames) {
319           // change .blog to .log
320           //
321           int n = strlen(file_arguments[i]);
322           int copy_len =
323             (n >= bin_ext_len ?
324                (strcmp(&file_arguments[i][n - bin_ext_len], LOG_FILE_BINARY_OBJECT_FILENAME_EXTENSION) == 0 ? n - bin_ext_len : n) :
325                n);
326 
327           char *out_filename = (char *)ats_malloc(copy_len + ascii_ext_len + 1);
328 
329           memcpy(out_filename, file_arguments[i], copy_len);
330           memcpy(&out_filename[copy_len], LOG_FILE_ASCII_OBJECT_FILENAME_EXTENSION, ascii_ext_len);
331           out_filename[copy_len + ascii_ext_len] = 0;
332 
333           out_fd = open_output_file(out_filename);
334           ats_free(out_filename);
335 
336           if (out_fd < 0) {
337             error = DATA_PROCESSING_ERROR;
338             continue;
339           }
340         }
341         if (follow_flag) {
342           lseek(in_fd, 0, SEEK_END);
343         }
344 
345         ino_t inode_num = get_inode_num(file_arguments[i]);
346         while (true) {
347           if (process_file(in_fd, out_fd) != 0) {
348             error = DATA_PROCESSING_ERROR;
349             break;
350           }
351           if (!follow_flag) {
352             break;
353           } else {
354             usleep(10000); // This avoids burning CPU, using poll() would have been nice, but doesn't work I think.
355 
356             // see if the file we're following has been rotated
357             if (access(file_arguments[i], F_OK) == 0) { // Sometimes there's a gap between logfile rotation and the actual presence
358                                                         // of a fresh file on disk. We must make sure we don't get caught in that
359                                                         // gap.
360               int fd = follow_rotate(file_arguments[i], inode_num);
361               if (fd == -1) {
362                 error = DATA_PROCESSING_ERROR;
363                 break;
364               } else if (fd > 0) {
365                 // we got a new fd to use
366                 Debug("logcat", "Detected logfile rotation. Following to new file");
367                 close(in_fd);
368                 in_fd = fd;
369 
370                 // update the inode number for the log file
371                 inode_num = get_inode_num(file_arguments[i]);
372               }
373             }
374           }
375         }
376       }
377 #if HAVE_POSIX_FADVISE
378       // Now that we're done reading a potentially large log file, we can tell the kernel that it's OK to evict
379       // the associated log file pages from cache
380       posix_fadvise(in_fd, 0, 0, POSIX_FADV_DONTNEED);
381 #endif
382     }
383   } else {
384     // read from stdin, allow STDIN to go EOF a few times until we get synced
385     //
386     int tries = 3;
387     while (--tries >= 0) {
388       if (process_file(STDIN_FILENO, out_fd) != 0) {
389         tries = -1;
390       }
391     }
392   }
393 
394   ::exit(error);
395 }
396