lindeas
/
jilo
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Fixes errors, better command-line options and progress output.

main
Yasen Pramatarov 2024-05-23 19:39:57 +03:00
parent f2ea57804b
commit b6602cee94
1 changed files with 78 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
jvb/jitsi-stats
View File

@ -11,11 +11,11 @@ DB="./jitsi-stats.db"
###
# SQLite queries
db_get_state='SELECT filename, filetime, position, inode FROM state WHERE id = 1;'
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', position='%s', inode='%s' WHERE id = 1;"
db_get_state='SELECT filename, filetime, filesize, position, inode FROM state WHERE id = 1;'
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE id = 1;"
db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');"
db_create="
DROP TABLE conferences;
DROP TABLE IF EXISTS conferences;
CREATE TABLE conferences (
id INTEGER PRIMARY_KEY,
conference_name TEXT,
@ -23,20 +23,23 @@ CREATE TABLE conferences (
start TEXT,
end TEXT
);
DROP TABLE state;
DROP TABLE IF EXISTS state;
CREATE TABLE state (
id INTEGER PRIMARY_KEY,
time TEXT,
filename TEXT,
filetime INTEGER,
filesize INTEGER,
position INTEGER CHECK(typeof(position)='integer'),
inode INTEGER
);
INSERT OR REPLACE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);"
INSERT OR REPLACE INTO state (id, time, filename, filetime, filesize, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
db_flush="
DELETE FROM conferences;
DELETE FROM state;
INSERT OR REPLACE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);"
INSERT OR REPLACE INTO state (id, time, filename, filetime, filesize, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]"
###
@ -51,9 +54,10 @@ get_state() {
set_state() {
local filename=$1
local filetime=$2
local position=${3:-0}
local inode=$4
db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$position" "$inode")
local filesize=$3
local position=${4:-0}
local inode=$5
db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode")
sqlite3 "$DB" "$db_set_state"
}
@ -64,17 +68,32 @@ jvb_log_parse() {
local file=$1
local start_pos=$2
local new_last_pos="$start_pos"
new_last_pos="$start_pos"
# Local assoc array for conference events tracking
declare -A start_times
# Get size and position for progress tracking
local total_size=$(stat -c '%s' "$file")
local processed_lines=0
local procesed_bytes=0
# We open the file and start reading from $start_pos bytes
exec 3<"$file"
while IFS= read -r line; do
# save new position (previous plus bytes in current line plus 1 for the new line)
new_last_pos=$((new_last_pos + ${#line} + 1))
# increment progress stats
processed_lines=$((processed_lines + 1))
processed_bytes=$((processed_bytes + ${#line} + 1))
# show progress if in verbose mode
if [[ "$verbose" == true ]]; then
local percent=$((100 * processed_bytes / total_size))
echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
fi
# locate conference starting event
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
timestamp="${BASH_REMATCH[1]}"
@ -97,36 +116,63 @@ jvb_log_parse() {
# We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
# Close the file descriptor and return the new position value
# Close the file descriptor
exec 3<&-
echo "$new_last_pos"
}
###
# commandline options
case "$1" in
--create-db|-d)
while getopts ":dfcpv" opt; do
case $opt in
d)
cmd="--create-db"
;;
f)
cmd="--flush"
;;
c)
cmd="--check"
;;
p)
cmd="--parse"
;;
v)
verbose=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
echo -e "$help"
exit 1
;;
esac
done
shift $((OPTIND -1))
case "$cmd" in
--create-db)
sqlite3 "$DB" "$db_create"
echo "Database created."
exit 0
;;
--flush|-f)
--flush)
sqlite3 "$DB" "$db_flush"
echo "Tables flushed."
exit 0
;;
--check|-c)
--check)
# Retrieve last log file and position inside it
IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state)
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)
# Initialize logfile vars
current_inode=$(stat -c '%i' "$LOGFILE")
current_filetime=$(stat -c '%Y' "$LOGFILE")
current_size=$(stat -c '%s' "$LOGFILE")
if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then
echo "It looks like a fresh install. You can now run log parsing."
@ -137,17 +183,19 @@ case "$1" in
echo "Last file: $last_file"
echo "Last filetime: $last_filetime"
echo "Last inode: $last_inode"
echo "Last size: $last_size"
echo "Last processed position: $last_pos"
echo "Current filetime: $current_filetime"
echo "Current inode: $current_inode"
echo "Current size: $current_size"
if [[ "$last_file" != "$LOGFILE" || "$last_inode" != "$current_inode" ]]; then
if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
echo "Log file has rotated."
else
echo "Log file has not rotated."
fi
if [[ "$current_filetime" -ne "$last_filetime" ]]; then
if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then
echo "New lines have been added to the log."
else
echo "No new lines in the log."
@ -155,44 +203,45 @@ case "$1" in
exit 0
;;
--parse|-p)
--parse)
# Retrieve last log file and position inside it
IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state)
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)
# Initialize logfile vars
last_pos=${last_pos:-0}
current_inode=$(stat -c '%i' "$LOGFILE")
current_filetime=$(stat -c '%Y' "$LOGFILE")
current_size=$(stat -c '%s' "$LOGFILE")
# Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate)
# parse the rotated log file
current_size=$(stat -c '%s' "$LOGFILE")
if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE"
last_pos=$(jvb_log_parse "$ROTATED_LOGFILE" 0)
jvb_log_parse "$ROTATED_LOGFILE" 0 "$verbose"
last_file="$ROTATED_LOGFILE"
last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
set_state "$last_file" "$last_filetime" "$last_pos" "$last_inode"
set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode"
fi
# parse the current log file
echo "Processing the current log file: $LOGFILE"
new_last_pos=$(jvb_log_parse "$LOGFILE" "$last_pos")
jvb_log_parse "$LOGFILE" "$last_pos" "$verbose"
# DEBUG
echo "New last position after parsing: $new_last_pos"
if [[ "$verbose" == true ]]; then
echo -e "\nNew last position after parsing: $new_last_pos"
fi
# update the state in db
set_state "$LOGFILE" "$current_filetime" "$new_last_pos" "$current_inode"
set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode"
echo "Data import finished."
exit 0
;;
*)
echo -e "Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p - parse the logs"
echo -e "$help"
exit 1
;;
esac