Ability to write out JSON data to a file for later collection.

master
Zed A. Shaw 2 years ago
parent afe3cfca47
commit ecf0bcc9b0
  1. 56
      02-filter-a-log-file/chains_parser.js

@ -243,10 +243,12 @@ const construct_request_chains = (by_ip, domain, as_set) => {
return ip_chains; return ip_chains;
} }
const sort_request_chains = (chains) => { const sort_request_chains = (chains, min) => {
const converted = []; const converted = [];
for(let [url, stats] of Object.entries(chains)) { for(let [url, stats] of Object.entries(chains)) {
if(stats.count < min) continue; // skip below min
if(stats.comes_from) { if(stats.comes_from) {
converted.push([stats.count, `[${stats.comes_from}] ${stats.full_chain.join(' ')}`]); converted.push([stats.count, `[${stats.comes_from}] ${stats.full_chain.join(' ')}`]);
} else { } else {
@ -257,14 +259,39 @@ const sort_request_chains = (chains) => {
return converted.sort((a, b) => b[0] - a[0]); return converted.sort((a, b) => b[0] - a[0]);
} }
const output_results = (chains_sorted, min) => { const output_results = async (stats, chains, format, outfile) => {
for(let [count, url] of chains_sorted) { if(format === "json") {
if(count >= min) { const data = {stats, chains, date: new Date()};
console.log(data);
} else {
const chains_sorted = sort_request_chains(chains, OPTS.min);
for(let [count, url] of chains_sorted) {
console.log(count, url); console.log(count, url);
} }
console.log(stats);
}
}
const write_results = async (stats, chains, format, outfile) => {
assert(outfile, "Output file required.");
// unlike unix APIs this uses exceptions rather than return values for errors
const fd = fs.openSync(outfile, "w+");
if(format === "json") {
const data = {stats, chains, date: new Date()};
const bytes = fs.writeSync(fd, Buffer.from(JSON.stringify(data, null, 4)), 0);
} else {
const chains_sorted = sort_request_chains(chains, OPTS.min);
for(let [count, url] of chains_sorted) {
const bytes = fs.writeSync(fd, Buffer.from(`${count} ${url}\n`));
}
} }
console.log(stats); fs.closeSync(fd);
} }
program program
@ -272,6 +299,7 @@ program
.option("--min <Number>", "The lowest count to print. Stop at this.", 1) .option("--min <Number>", "The lowest count to print. Stop at this.", 1)
.option("--errors", "Show the erorrs so you can fix them.", false) .option("--errors", "Show the erorrs so you can fix them.", false)
.option("--format <string>", "Output format, text or json. Ignores min for raw output.", "json") .option("--format <string>", "Output format, text or json. Ignores min for raw output.", "json")
.option("--outfile <string>", "Save to file rather than stdout.")
.requiredOption("--domain <String>", "Domain for the log. Gets removed as a refer.") .requiredOption("--domain <String>", "Domain for the log. Gets removed as a refer.")
.requiredOption("--input <String>", "Input file.") .requiredOption("--input <String>", "Input file.")
.description("Processes different web server logs to determine request chain frequency.") .description("Processes different web server logs to determine request chain frequency.")
@ -283,12 +311,16 @@ OPTS.min = parseInt(OPTS.min);
assert(!isNaN(OPTS.min), `min must be a number, you have ${OPTS.min}`); assert(!isNaN(OPTS.min), `min must be a number, you have ${OPTS.min}`);
const [by_ip, stats] = await parse_logs(OPTS.input, OPTS.errors); try {
const chains = construct_request_chains(by_ip, OPTS.domain, OPTS.set); const [by_ip, stats] = await parse_logs(OPTS.input, OPTS.errors);
const chains_sorted = sort_request_chains(chains); const chains = construct_request_chains(by_ip, OPTS.domain, OPTS.set);
if(OPTS.format === "json") { if(OPTS.outfile) {
console.log(chains); write_results(stats, chains, OPTS.format, OPTS.outfile);
} else { } else {
output_results(chains_sorted, OPTS.min); output_results(stats, chains, OPTS.format, OPTS.outfile);
}
} catch(error) {
console.error(error.message);
process.exit(1);
} }

Loading…
Cancel
Save