jilo/jilo

526 lines
20 KiB
Bash

#!/usr/bin/env bash
###
# JItsi Log Observer
#
# Bash script for Jitsi Meet components (Videobridge, Jicofo, etc.) logs parsing
###
### Configuration file (overrides default configs)
CONFIG_FILE="./jilo.conf"
### Default configuration
# default log files and processes
DEFAULT_JVB_LOGFILE="/var/log/jitsi/jvb.log"
DEFAULT_JVB_PROCESS="videobridge"
DEFAULT_JICOFO_LOGFILE="/var/log/jitsi/jicofo.log"
DEFAULT_JICOFO_PROCESS="jicofo"
# Default database type (sqlite, mysql|mariadb)
DEFAULT_DB_TYPE="sqlite"
# Default SQLite database file
DEFAULT_DB="./jilo.db"
# Default MySQL/MariaDB configuration
DEFAULT_MYSQL_HOST="localhost"
DEFAULT_MYSQL_USER="jilo"
DEFAULT_MYSQL_PASS="jilopass"
DEFAULT_MYSQL_DB="jilo_db"
# Load configurations from the config file if it exists
if [[ -f "$CONFIG_FILE" ]]; then
source "$CONFIG_FILE"
fi
# use default values if not overriden by config file
JVB_LOGFILE=${JVB_LOGFILE:-$DEFAULT_JVB_LOGFILE}
JVB_PROCESS=${JVB_PROCESS:-$DEFAULT_JVB_PROCESS}
JICOFO_LOGFILE=${JICOFO_LOGFILE:-$DEFAULT_JICOFO_LOGFILE}
JICOFO_PROCESS=${JICOFO_PROCESS:-$DEFAULT_JICOFO_PROCESS}
DB=${DB:-$DEFAULT_DB}
DB_TYPE=${DB_TYPE:-$DEFAULT_DB_TYPE}
MYSQL_HOST=${MYSQL_HOST:-$DEFAULT_MYSQL_HOST}
MYSQL_USER=${MYSQL_USER:-$DEFAULT_MYSQL_USER}
MYSQL_PASS=${MYSQL_PASS:-$DEFAULT_MYSQL_PASS}
MYSQL_DB=${MYSQL_DB:-$DEFAULT_MYSQL_DB}
###
# DB queries
db_get_state_template="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = '%s';"
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = '%s';"
db_insert_conference_template="INSERT INTO conferences (jitsi_component, conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s', '%s');"
db_insert_participant_template="INSERT INTO participants (conference_id, event_time, event_type, endpoint_id, stats_id, participant_ip) VALUES ('%s', '%s', '%s', '%s', '%s', '%s');"
db_drop="
DROP TABLE IF EXISTS conferences;
DROP TABLE IF EXISTS participants;
DROP TABLE IF EXISTS state;"
db_create="CREATE TABLE conferences (
id INTEGER PRIMARY_KEY,
jitsi_component TEXT,
conference_name TEXT,
conference_id TEXT,
start TEXT,
end TEXT
);
CREATE TABLE participants (
id INTEGER PRIMARY_KEY,
conference_id INTEGER,
event_time TEXT,
event_type TEXT,
endpoint_id TEXT,
stats_id TEXT,
participant_ip TEXT
);
CREATE TABLE state (
id INTEGER PRIMARY_KEY,
jitsi_component TEXT,
time TEXT,
filename TEXT,
filetime INTEGER,
filesize INTEGER,
position INTEGER CHECK(typeof(position)='integer'),
inode INTEGER
);"
db_init="
INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JVB', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);
INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (2, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
db_flush="
DELETE FROM conferences;
DELETE FROM participants;
DELETE FROM state;"
help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]"
###
# First we check for requirements
check_requirements() {
# required programs, anything non-bash - edit as needed
# deb packages - sqlite3, coreutils(stat,dd)
local required_programs=("sqlite3" "stat" "dd")
local requirements_missing=''
for program in "${required_programs[@]}"; do
if ! command -v "$program" &> /dev/null; then
requirements_missing+="$program, "
fi
done
if [[ "$requirements_missing" != '' ]]; then
requirements_missing=${requirements_missing::-2}
echo "Error: $requirements_missing - not found. Please install to proceed."
fi
}
check_requirements
###
# DB functions for Sqlite3 and for MySQL/MariaDB
# normalize DB schemas for Sqlite3 and MySQL/MariaDB in order to compare them when needed
db_normalize_schema() {
echo "$1" | tr -d '\n' | tr -s ' ' | tr ',' '\n' | sort
}
# execute a query and return the result
db_query() {
local query=$1
if [[ "$DB_TYPE" == "sqlite" ]]; then
sqlite3 "$DB" "$query"
elif [[ "$DB_TYPE" == "mysql" || "$DB_TYPE" == "mariadb" ]]; then
mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p "$MYSQL_PASS" -D "$MYSQL_DB" -se "$query"
else
echo "Error: unknown database type $DB_TYPE."
exit 1
fi
}
# Get the last processed state from the database
get_state() {
db_get_state=$(printf "$db_get_state_template" "$1")
db_query "$db_get_state"
}
# Update the state database
set_state() {
local filename=$1
local filetime=$2
local filesize=$3
local position=${4:-0}
local inode=$5
local jitsi_component=$6
db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode" "$jitsi_component")
db_query "$db_set_state"
}
###
# Main parsing funstion
jitsi_log_parse() {
local file=$1
local start_pos=$2
new_last_pos="$start_pos"
# Local assoc array for conference events tracking
declare -A start_times
# Get size and position for progress tracking
local total_size
total_size=$(stat -c '%s' "$file")
local processed_lines=0
local processed_bytes=0
# We open the file and start reading from $start_pos bytes
exec 3<"$file"
while IFS= read -r line; do
# save new position (previous plus bytes in current line plus 1 for the new line)
new_last_pos=$((new_last_pos + ${#line} + 1))
# increment progress stats
processed_lines=$((processed_lines + 1))
processed_bytes=$((processed_bytes + ${#line} + 1))
# show progress if in verbose mode
if [[ "$verbose" == true ]]; then
local percent=$((100 * processed_bytes / total_size))
echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
fi
case $jitsi_component in
JVB)
# locate conference starting event
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
timestamp="${BASH_REMATCH[1]}"
conference_id="${BASH_REMATCH[2]}"
start_times["$conference_id"]="$timestamp"
# locate participant joining event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=${conference_id}\ .*epId=([a-zA-Z0-9-]+)\ stats_id=([a-zA-Z0-9-]+)\ .*Starting\ the\ Agent\ without\ remote\ candidates ]]; then
event_time="${BASH_REMATCH[1]}"
event_type='participant joining'
participant_endpoint_id="${BASH_REMATCH[2]}"
participant_stats_id="${BASH_REMATCH[3]}"
if [[ -n "$conference_id" ]]; then
db_insert=$(printf "$db_insert_participant_template" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "")
db_query "$db_insert"
fi
# locate participant pair selection event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=${conference_id}\ .*epId=${participant_endpoint_id}\ stats_id=${participant_stats_id}\ .*Selected\ pair\ for\ stream\ .*([0-9.]+):10000/udp/srflx\ \-\>\ ([0-9.]+):[0-9]+/udp/prflx ]]; then
event_time="${BASH_REMATCH[1]}"
event_type='pair selected'
participant_IP="${BASH_REMATCH[3]}"
if [[ -n "$conference_id" ]]; then
db_insert=$(printf "$db_insert_participant_template" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "$participant_IP")
db_query "$db_insert"
fi
# locate participant leaving event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=${conference_id}\ .*epId=${participant_endpoint_id}\ stats_id=${participant_stats_id}\]\ Endpoint\.expire.*:\ Expired\. ]]; then
event_time="${BASH_REMATCH[1]}"
event_type='participant leaving'
if [[ -n "$conference_id" ]]; then
db_insert=$(printf "$db_insert_participant_template" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "$participant_IP")
db_query "$db_insert"
# the participant left, forget about him
unset $event_time $event_type $participant_endpoint_id $participant_stats_id $participant_IP
fi
# locate the corresponding conference ending event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then
end_time="${BASH_REMATCH[1]}"
conference_id="${BASH_REMATCH[2]}"
conference_name="${BASH_REMATCH[3]}"
start_time="${start_times["$conference_id"]}"
if [[ -n "$start_time" ]]; then
db_insert=$(printf "$db_insert_conference_template" "$jitsi_component" "$conference_name" "$conference_id" "$start_time" "$end_time")
db_query "$db_insert"
# the conference ended, forget about it
unset "start_times[$conference_id]" $start_time $end_time $conference_id $conference_name
fi
fi
;;
JICOFO)
# locate conference starting event
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then
timestamp="${BASH_REMATCH[1]}"
conference_name="${BASH_REMATCH[2]}"
start_times["$conference_name"]="$timestamp"
# locate the corresponding conference ending event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then
end_time="${BASH_REMATCH[1]}"
conference_name="${BASH_REMATCH[2]}"
conference_id="${BASH_REMATCH[3]}"
start_time="${start_times["$conference_name"]}"
if [[ -n "$start_time" ]]; then
db_insert=$(printf "$db_insert_conference_template" "$jitsi_component" "$conference_name" "$conference_id" "$start_time" "$end_time")
db_query "$db_insert"
unset "start_times[$conference_name]"
fi
fi
;;
esac
# We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
# Close the file descriptor
exec 3<&-
}
### FIXME - this is not currently used
# check if and which process is running
is_process_running() {
pgrep -f "$1" >/dev/null 2>&1
}
# commandline options
while getopts ":dfcpv" opt; do
case $opt in
d)
cmd="--create-db"
;;
f)
cmd="--flush"
;;
c)
cmd="--check"
;;
p)
cmd="--parse"
;;
v)
verbose=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
echo -e "$help"
exit 1
;;
esac
done
shift $((OPTIND -1))
case "$cmd" in
--create-db)
db_query "$db_drop"
db_query "$db_create"
db_query "$db_init"
echo "Database created."
exit 0
;;
--flush)
db_query "$db_flush"
db_query "$db_init"
echo "Tables flushed."
exit 0
;;
--check)
# database checks
if [[ "$DB_TYPE" == "sqlite" ]]; then
# First check if database exists
if [[ ! -f "$DB" ]]; then
echo "Database not found. If it's a fresh install, please install the database first."
exit 1
fi
# get current and expected db schemas in comparable format
current_db_schema=$(sqlite3 "$DB" .schema)
current_db_schema_normalized=$(db_normalize_schema "$current_db_schema")
expected_db_schema_normalized=$(db_normalize_schema "$db_create")
# compare the DB schema to the expected one
if [[ "$current_db_schema_normalized" != "$expected_db_schema_normalized" ]]; then
echo "The database doesn't match the expected schema. Please check it, and if needed, reinstall it."
exit 1
fi
elif [[ "$DB_TYPE" == "mysql" || "$DB_TYPE" == "mariadb" ]]; then
# First check if database exists
if ! mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p "$MYSQL_PASS" -e "USE $MYSQL_DB"; then
echo "Database not found. If it's a fresh install, please install the database first."
exit 1
fi
# Get the list of tables, omiting the 'show tables' header
tables=()
while IFS= read -r line; do
tables+=("$line")
done < <(mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p "$MYSQL_PASS" -D "$MYSQL_DB" -e "SHOW TABLES;" | tail -n +2)
# get current and expected db schemas in comparable format
current_db_schema=''
for table in ${tables[@]}; do
create_table_string=$(mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p "$MYSQL_PASS" -D "$MYSQL_DB" -e "SHOW CREATE TABLE $table\G" | grep -v "Table" | grep -v "Create Table")
create_table_string="${create_table_string#"${create_table_string%%[^[:space:]]*}"}" # remove leading spaces"
current_db_schema+="$create_table_string"
done
current_db_schema_normalized=$(db_normalize_schema "$current_db_schema")
expected_db_schema_normalized=$(db_normalize_schema "$db_create")
# compare the DB schema to the expected one
if [[ "$current_db_schema_normalized" != "$expected_db_schema_normalized" ]]; then
echo "The database doesn't match the expected schema. Please check it, and if needed, reinstall it."
exit 1
fi
else
echo "Error: unknown database type $DB_TYPE."
exit 1
fi
# Check if log files exist
jvb_found=false
jicofo_found=false
if [[ -f "$JVB_LOGFILE" ]]; then
jvb_found=true
jitsi_components+=('JVB')
fi
if [[ -f "$JICOFO_LOGFILE" ]]; then
jicofo_found=true
jitsi_components+=('JICOFO')
fi
# if no logs present, exit
if [[ "$jvb_found" == false && "$jicofo_found" == false ]]; then
echo "Neither \"$JVB_PROCESS\" ($JVB_LOGFILE) nor \"$JICOFO_PROCESS\" ($JICOFO_LOGFILE) log files are found."
exit 1
else
# otherwise loop through the found components
for jitsi_component in "${jitsi_components[@]}"; do
# Retrieve last log file and position inside it
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< "$(get_state "$jitsi_component")"
# Initialize logfile vars
LOGFILE=$(eval "echo \$${jitsi_component}_LOGFILE")
ROTATED_LOGFILE="$LOGFILE.1"
current_inode=$(stat -c '%i' "$LOGFILE")
current_filetime=$(stat -c '%Y' "$LOGFILE")
current_size=$(stat -c '%s' "$LOGFILE")
if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then
echo "It looks like a fresh install. You can now run log parsing."
exit 0
fi
# report
echo "Last file: $last_file"
echo "Last filetime: $last_filetime"
echo "Last inode: $last_inode"
echo "Last size: $last_size"
echo "Last processed position: $last_pos"
echo "Current filetime: $current_filetime"
echo "Current inode: $current_inode"
echo "Current size: $current_size"
if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
echo "Log file has rotated."
else
echo "Log file has not rotated."
fi
if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then
echo -e "New lines have been added to the log.\n"
else
echo -e "No new lines in the log.\n"
fi
done
fi
exit 0
;;
--parse)
# Check if log files exist
jvb_found=false
jicofo_found=false
if [[ -f "$JVB_LOGFILE" ]]; then
jvb_found=true
jitsi_components+=('JVB')
fi
if [[ -f "$JICOFO_LOGFILE" ]]; then
jicofo_found=true
jitsi_components+=('JICOFO')
fi
# if no logs present, exit
if [[ "$jvb_found" == false && "$jicofo_found" == false ]]; then
echo "Neither \"$JVB_PROCESS\" ($JVB_LOGFILE) nor \"$JICOFO_PROCESS\" ($JICOFO_LOGFILE) log files are found."
exit 1
else
# otherwise loop through the found components
for jitsi_component in "${jitsi_components[@]}"; do
# Retrieve last log file and position inside it
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< "$(get_state "$jitsi_component")"
# Initialize logfile vars
LOGFILE=$(eval "echo \$${jitsi_component}_LOGFILE")
ROTATED_LOGFILE="$LOGFILE.1"
last_pos=${last_pos:-0}
current_inode=$(stat -c '%i' "$LOGFILE")
current_filetime=$(stat -c '%Y' "$LOGFILE")
current_size=$(stat -c '%s' "$LOGFILE")
# Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate)
# parse the rotated log file
if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE"
jitsi_log_parse "$ROTATED_LOGFILE" 0 "$verbose"
last_file="$ROTATED_LOGFILE"
last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode" "$jitsi_component"
fi
# parse the current log file
echo "Processing the current log file: $LOGFILE"
jitsi_log_parse "$LOGFILE" "$last_pos" "$verbose"
if [[ "$verbose" == true ]]; then
if [[ "$new_last_pos" == "$last_pos" ]]; then
echo "The file has not changed, no new data inserted."
else
echo -e "\nNew last position after parsing: $new_last_pos"
fi
fi
# update the state in db
set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode" "$jitsi_component"
done
fi
echo "Data import finished."
exit 0
;;
*)
echo -e "$help"
exit 1
;;
esac