Break out the reporting from the parsing so I can parse the files on all my servers in a cron job, but collect them and report on a central server.

02-filter-a-log-file/chains_parser.js

@@ -9,8 +9,7 @@ import format from "date-fns/format/index.js";
 program
   .option("--min <Number>", "The lowest count to print. Stop at this.", 1)
   .option("--errors", "Show the errors so you can fix them.", false)
-  .option("--format <String>", "Output format, text or json. Ignores min for raw output.")
+  .requiredOption("--outfile <String>", "Save to file rather than stdout.")
-  .option("--outfile <String>", "Save to file rather than stdout.")
   .requiredOption("--domain <String>", "Domain for the log. Gets removed as a refer.")
   .requiredOption("--input <String>", "Input file glob.")
   .description("Processes different web server logs to determine request chain frequency.")
@@ -261,60 +260,22 @@ const construct_request_chains = (by_ip, domain) => {
   return ip_chains;
 }
 
-const sort_request_chains = (chains, min) => {
-  const converted = [];
-
-  for(let [url, stats] of Object.entries(chains)) {
-    if(stats.count < min) continue; // skip below min
-
-    if(stats.comes_from) {
-      converted.push([stats.count, `[${stats.comes_from}] ${stats.full_chain.join(' ')}`]);
-    } else {
-      converted.push([stats.count, `${stats.full_chain.join(' ')}`]);
-    }
-  }
-
-  return converted.sort((a, b) => b[0] - a[0]);
-}
-
-const output_report = async (stats, data, domain, min) => {
-  for(let key in data) {
-    console.log(`--- ${key} ---`);
-    const chains = construct_request_chains(data[key], domain);
-    const chains_sorted = sort_request_chains(chains, min);
-
-    for(let [count, url] of chains_sorted) {
-      console.log(count, url);
-    }
-  }
-}
-
 const output_json = async (stats, data, domain, min, outfile) => {
   const result = {};
 
   for(let key in data) {
-    const chains = construct_request_chains(data[key], domain);
+    result[key] = construct_request_chains(data[key], domain);
-    const chains_sorted = sort_request_chains(chains, min);
-    result[key] = chains_sorted;
   }
 
-  if(outfile) {
   const fd = fs.openSync(outfile, "w+");
-    const data = {stats, result, generated_on: new Date()};
+  const output = {stats, domain, result, generated_on: new Date()};
-    const bytes = fs.writeSync(fd, Buffer.from(JSON.stringify(data, null, 4)), 0);
+  const bytes = fs.writeSync(fd, Buffer.from(JSON.stringify(output)), 0);
   fs.closeSync(fd);
-  } else {
-    console.log({stats, result});
-  }
 }
 
 
 const [by_date, stats] = await parse_logs_glob(OPTS.input, OPTS.errors);
 
-if(OPTS.format === "json") {
 output_json(stats, by_date, OPTS.domain, OPTS.min, OPTS.outfile);
-} else {
-  output_report(stats, by_date, OPTS.domain, OPTS.min);
-}
 
 console.log(stats);

02-filter-a-log-file/chains_reporter.js

@@ -0,0 +1,49 @@
+import fs from "fs";
+import assert from "assert";
+import readline from "readline";
+import { program } from "commander";
+import glob from "fast-glob";
+import format from "date-fns/format/index.js";
+
+program
+  .option("--min <Number>", "The lowest count to print. Stop at this.", 1)
+  .option("--errors", "Show the errors so you can fix them.", false)
+  .option("--outfile <String>", "Save to file rather than stdout.")
+  .requiredOption("--input <String>", "Input file glob.")
+  .description("Loads the output of a chain parser and produces a report.")
+  .version(0.1);
+
+program.parse();
+const OPTS = program.opts();
+OPTS.min = parseInt(OPTS.min);
+assert(!isNaN(OPTS.min), `min must be a number, you have ${OPTS.min}`);
+
+const sort_request_chains = (chains, min) => {
+  const converted = [];
+
+  for(let [url, stats] of Object.entries(chains)) {
+    if(stats.count < min) continue; // skip below min
+
+    if(stats.comes_from) {
+      converted.push([stats.count, `[${stats.comes_from}] ${stats.full_chain.join(' ')}`]);
+    } else {
+      converted.push([stats.count, `${stats.full_chain.join(' ')}`]);
+    }
+  }
+
+  return converted.sort((a, b) => b[0] - a[0]);
+}
+
+const { stats, domain, result, generated_on} = JSON.parse(fs.readFileSync(OPTS.input));
+
+const dates = Object.keys(result).sort();
+
+for(let key of dates) {
+  console.log(`--- ${key} ---`);
+
+  const stats = sort_request_chains(result[key], OPTS.min);
+
+  for(let [count, chain] of stats) {
+    console.log(count, "\t", chain);
+  }
+}