Ability to write out JSON data to a file for later collection.

02-filter-a-log-file/chains_parser.js

@@ -243,10 +243,12 @@ const construct_request_chains = (by_ip, domain, as_set) => {
   return ip_chains;
 }
 
-const sort_request_chains = (chains) => {
+const sort_request_chains = (chains, min) => {
   const converted = [];
 
   for(let [url, stats] of Object.entries(chains)) {
+    if(stats.count < min) continue; // skip below min
+
     if(stats.comes_from) {
       converted.push([stats.count, `[${stats.comes_from}] ${stats.full_chain.join(' ')}`]);
     } else {
@@ -257,14 +259,39 @@ const sort_request_chains = (chains) => {
   return converted.sort((a, b) => b[0] - a[0]);
 }
 
-const output_results = (chains_sorted, min) => {
+const output_results = async (stats, chains, format, outfile) => {
-  for(let [count, url] of chains_sorted) {
+  if(format === "json") {
-    if(count >= min) {
+    const data = {stats, chains, date: new Date()};
+    console.log(data);
+  } else {
+    const chains_sorted = sort_request_chains(chains, OPTS.min);
+
+    for(let [count, url] of chains_sorted) {
       console.log(count, url);
     }
+
+    console.log(stats);
+  }
+}
+
+const write_results = async (stats, chains, format, outfile) => {
+  assert(outfile, "Output file required.");
+
+  // unlike unix APIs this uses exceptions rather than return values for errors
+  const fd = fs.openSync(outfile, "w+");
+
+  if(format === "json") {
+    const data = {stats, chains, date: new Date()};
+    const bytes = fs.writeSync(fd, Buffer.from(JSON.stringify(data, null, 4)), 0);
+  } else {
+    const chains_sorted = sort_request_chains(chains, OPTS.min);
+
+    for(let [count, url] of chains_sorted) {
+      const bytes = fs.writeSync(fd, Buffer.from(`${count} ${url}\n`));
+    }
   }
 
-  console.log(stats);
+  fs.closeSync(fd);
 }
 
 program
@@ -272,6 +299,7 @@ program
   .option("--min <Number>", "The lowest count to print. Stop at this.", 1)
   .option("--errors", "Show the erorrs so you can fix them.", false)
   .option("--format <string>", "Output format, text or json. Ignores min for raw output.", "json")
+  .option("--outfile <string>", "Save to file rather than stdout.")
   .requiredOption("--domain <String>", "Domain for the log. Gets removed as a refer.")
   .requiredOption("--input <String>", "Input file.")
   .description("Processes different web server logs to determine request chain frequency.")
@@ -283,12 +311,16 @@ OPTS.min = parseInt(OPTS.min);
 
 assert(!isNaN(OPTS.min), `min must be a number, you have ${OPTS.min}`);
 
-const [by_ip, stats] = await parse_logs(OPTS.input, OPTS.errors);
+try {
-const chains = construct_request_chains(by_ip, OPTS.domain, OPTS.set);
+  const [by_ip, stats] = await parse_logs(OPTS.input, OPTS.errors);
-const chains_sorted = sort_request_chains(chains);
+  const chains = construct_request_chains(by_ip, OPTS.domain, OPTS.set);
 
-if(OPTS.format === "json") {
+  if(OPTS.outfile) {
-  console.log(chains);
+    write_results(stats, chains, OPTS.format, OPTS.outfile);
-} else {
+  } else {
-  output_results(chains_sorted, OPTS.min);
+    output_results(stats, chains, OPTS.format, OPTS.outfile);
+  }
+} catch(error) {
+  console.error(error.message);
+  process.exit(1);
 }