You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
261 lines
5.7 KiB
261 lines
5.7 KiB
2 years ago
|
import fs from "fs";
|
||
|
import assert from "assert";
|
||
|
import UAParser from "ua-parser-js";
|
||
|
import readline from "readline";
|
||
2 years ago
|
import { program } from "commander";
|
||
2 years ago
|
|
||
|
class Parser {
|
||
|
constructor() {
|
||
|
this.text = "";
|
||
|
this.next = "";
|
||
|
}
|
||
|
|
||
2 years ago
|
match(reg, consume=true) {
|
||
2 years ago
|
const n = this.next.match(reg);
|
||
|
|
||
2 years ago
|
if(n === null) {
|
||
|
return undefined;
|
||
|
} else {
|
||
|
const element = n.length > 1 ? n.slice(1) : n[0];
|
||
|
|
||
|
if(consume) {
|
||
|
this.next = this.next.slice(n[0].length);
|
||
|
}
|
||
2 years ago
|
|
||
2 years ago
|
return element;
|
||
|
}
|
||
2 years ago
|
}
|
||
|
|
||
|
start(line) {
|
||
|
this.text = line;
|
||
|
this.next = line;
|
||
|
}
|
||
|
|
||
2 years ago
|
ws(consume=true) {
|
||
|
return this.match(/ +/, consume);
|
||
2 years ago
|
}
|
||
|
|
||
2 years ago
|
parse_new_log(ip) {
|
||
|
const [port, conn_id, conn_count] = this.match(/^([0-9]+):([0-9]+):([0-9]+)/);
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const [ time ] = this.match(/\[(.*)\]/);
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const [ full_url ] = this.match(/"(.+?)"/);
|
||
2 years ago
|
|
||
2 years ago
|
const [url, params] = full_url.split("?");
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const code = this.match(/\-|[0-9]+/);
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const bytes = this.match(/\-|[0-9]+/);
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const [refer] = this.match(/"(.+?)"/);
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const [ua] = this.match(/"(.+?)"/);
|
||
|
|
||
|
return {
|
||
|
ip,
|
||
|
conn_id: parseInt(conn_id),
|
||
|
conn_count: parseInt(conn_count),
|
||
|
time: new Date(time),
|
||
|
url, params,
|
||
|
code: parseInt(code),
|
||
|
size: parseInt(bytes),
|
||
|
refer: refer === '-' ? undefined : refer,
|
||
|
ua: UAParser(ua)
|
||
|
};
|
||
2 years ago
|
}
|
||
|
|
||
2 years ago
|
parse_old_log(ip) {
|
||
|
this.match(/- -/);
|
||
|
|
||
|
this.ws();
|
||
|
|
||
|
// FORMAT: 29/Mar/2022:22:40:52 +0200
|
||
|
const [ day, month, year, hour, minute, seconds, tz_offset ] = this.match(/\[([0-9]+)\/([A-Za-z]+)\/([0-9]+):([0-9]+):([0-9]+):([0-9]+) (.+?)\]/);
|
||
|
|
||
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const [ method, full_url, http_version ] = this.match(/"([A-Z]+) (.+) HTTP\/([0-9].[0-9])"/);
|
||
2 years ago
|
|
||
2 years ago
|
const [ url, params ] = full_url.split("?");
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const code = this.match(/\-|[0-9]+/);
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const bytes = this.match(/\-|[0-9]+/);
|
||
2 years ago
|
|
||
2 years ago
|
this.ws();
|
||
2 years ago
|
|
||
2 years ago
|
const [refer] = this.match(/"(.+?)"/);
|
||
|
|
||
|
this.ws();
|
||
|
|
||
|
const [ua] = this.match(/"(.+?)"/);
|
||
|
|
||
|
// this is another IP address sometimes in another log format that I'll ignore
|
||
|
const unknown = this.match(/".+?"$/);
|
||
|
|
||
|
return {
|
||
|
ip,
|
||
|
method,
|
||
|
http_version,
|
||
|
time: new Date(`${day} ${month} ${year} ${hour}:${minute}:${seconds} ${tz_offset}`),
|
||
|
url, params,
|
||
|
code: parseInt(code),
|
||
|
size: parseInt(bytes),
|
||
|
refer: refer === '-' ? undefined : refer,
|
||
|
ua: UAParser(ua)
|
||
|
};
|
||
|
}
|
||
|
|
||
|
parse() {
|
||
|
const ip = this.match(/^[0-9\.]+/);
|
||
|
const test = this.match(/(:| )/);
|
||
|
|
||
|
// BUG: uhh for some reason it needs == here? === says : doesn't equal :
|
||
|
if(test == ":") {
|
||
|
return this.parse_new_log(ip);
|
||
|
} else if(test == " ") {
|
||
|
return this.parse_old_log(ip);
|
||
|
} else {
|
||
|
// console.log(`PARSE ERROR, expected : or ' ' but got ${test}`);
|
||
|
return {};
|
||
2 years ago
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
2 years ago
|
const parse_logs = async (file_name, errors) => {
|
||
2 years ago
|
const read_stream = fs.createReadStream(file_name);
|
||
2 years ago
|
|
||
2 years ago
|
const rl = readline.createInterface({
|
||
|
input: read_stream,
|
||
|
crlfDelay: Infinity
|
||
|
});
|
||
2 years ago
|
|
||
2 years ago
|
const parser = new Parser();
|
||
2 years ago
|
|
||
2 years ago
|
const stats = {
|
||
|
lines: 0,
|
||
|
chains: 0,
|
||
|
excluded: 0,
|
||
|
errors: 0,
|
||
|
roots: 0,
|
||
|
firsts: 0
|
||
|
};
|
||
|
|
||
2 years ago
|
const skip = /(authcheck|.*\.svg|.*\.webmanifest|.*\.js|.*\.css|.*php|socket\.io|\.env|.*\.png|.*\.txt|.*\.woff|.*\.jpg|.*\.mp4|.*\.torrent|\-|.*\.ico|\/api\/.*\?.*|.*\.html|.*\.map|.*.php)/
|
||
|
|
||
2 years ago
|
const by_ip = {};
|
||
|
|
||
2 years ago
|
for await (let line of rl) {
|
||
|
try {
|
||
|
stats.lines += 1;
|
||
|
parser.start(line);
|
||
|
|
||
|
const data = parser.parse();
|
||
|
|
||
|
// skip lines that have content we don't care about
|
||
|
if(data.url.match(skip)) continue;
|
||
|
|
||
2 years ago
|
// store or update the chain in the by_ip chain
|
||
2 years ago
|
const ip_chain = by_ip[data.ip] || [];
|
||
2 years ago
|
|
||
2 years ago
|
ip_chain.push(data);
|
||
2 years ago
|
|
||
2 years ago
|
by_ip[data.ip] = ip_chain;
|
||
2 years ago
|
} catch(error) {
|
||
2 years ago
|
if(errors) console.error(error);
|
||
|
|
||
2 years ago
|
stats.errors += 1;
|
||
2 years ago
|
}
|
||
2 years ago
|
}
|
||
|
|
||
2 years ago
|
return [by_ip, stats];
|
||
2 years ago
|
}
|
||
|
|
||
2 years ago
|
const chain_to_set = (requests) => {
|
||
|
const path = new Set();
|
||
2 years ago
|
|
||
|
for(let r of requests) {
|
||
2 years ago
|
path.add(r.url);
|
||
2 years ago
|
}
|
||
|
|
||
|
return path.values();
|
||
|
}
|
||
|
|
||
2 years ago
|
const chain_to_list = (requests) => {
|
||
|
const path = [];
|
||
|
let seen;
|
||
|
|
||
|
for(let r of requests) {
|
||
|
if(r.url != seen) {
|
||
|
path.push(r.url);
|
||
|
seen = r.url;
|
||
|
}
|
||
2 years ago
|
}
|
||
2 years ago
|
|
||
|
return path.values();
|
||
2 years ago
|
}
|
||
|
|
||
2 years ago
|
const sort_request_chains = (by_ip, as_set) => {
|
||
2 years ago
|
let ip_chains = {};
|
||
2 years ago
|
let seen;
|
||
2 years ago
|
|
||
2 years ago
|
for(let [ip, requests] of Object.entries(by_ip)) {
|
||
2 years ago
|
const chain = as_set ? chain_to_set(requests) : chain_to_list(requests);
|
||
2 years ago
|
|
||
2 years ago
|
const ref = requests[0].refer ? `[${requests[0].refer}]` : "";
|
||
|
const url_set = [ref, ...chain].join(" ");
|
||
2 years ago
|
|
||
2 years ago
|
ip_chains[url_set] = url_set in ip_chains ? ip_chains[url_set] + 1 : 1;
|
||
2 years ago
|
}
|
||
|
|
||
2 years ago
|
const chains_sorted = Object.entries(ip_chains);
|
||
|
chains_sorted.sort((a, b) => b[1] - a[1]);
|
||
2 years ago
|
|
||
2 years ago
|
return chains_sorted;
|
||
|
}
|
||
2 years ago
|
|
||
2 years ago
|
const output_results = (min, chains_sorted) => {
|
||
|
for(let [url, count] of chains_sorted) {
|
||
|
if(count >= min) {
|
||
|
console.log(count, url);
|
||
|
}
|
||
2 years ago
|
}
|
||
2 years ago
|
|
||
2 years ago
|
console.log(stats);
|
||
2 years ago
|
}
|
||
2 years ago
|
|
||
2 years ago
|
program
|
||
|
.option("--no-set", "Use a Set instead of a list for chains.")
|
||
|
.option("--min <Number>", "The lowest count to print. Stop at this.", 1)
|
||
|
.option("--errors", "Show the erorrs so you can fix them.", false)
|
||
|
.requiredOption("--input <String>", "Input file.")
|
||
|
.description("Processes different web server logs to determine request chain frequency.")
|
||
|
.version(0.1);
|
||
2 years ago
|
|
||
2 years ago
|
program.parse();
|
||
|
const OPTS = program.opts();
|
||
|
OPTS.min = parseInt(OPTS.min);
|
||
2 years ago
|
|
||
2 years ago
|
assert(!isNaN(OPTS.min), `min must be a number, you have ${OPTS.min}`);
|
||
2 years ago
|
|
||
2 years ago
|
const [by_ip, stats] = await parse_logs(OPTS.input, OPTS.errors);
|
||
|
const chains_sorted = sort_request_chains(by_ip, OPTS.set);
|
||
|
output_results(OPTS.min, chains_sorted);
|