843 lines
37 KiB
Bash
843 lines
37 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
###
|
|
# JItsi Log Observer
|
|
#
|
|
# Bash script for Jitsi Meet components (Videobridge, Jicofo, etc.) logs parsing
|
|
###
|
|
|
|
VERSION="0.1.1"
|
|
RELEASE_DATE="2024-06-18"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" #"
|
|
|
|
### Configuration file (overrides default configs)
|
|
CONFIG_FILE="$SCRIPT_DIR/jilo.conf"
|
|
|
|
### Default configuration
|
|
|
|
# default log files and processes
|
|
DEFAULT_JVB_LOGFILE="/var/log/jitsi/jvb.log"
|
|
DEFAULT_JVB_PROCESS="videobridge"
|
|
DEFAULT_JICOFO_LOGFILE="/var/log/jitsi/jicofo.log"
|
|
DEFAULT_JICOFO_PROCESS="jicofo"
|
|
# Default database type (sqlite, mysql|mariadb)
|
|
DEFAULT_DB_TYPE="sqlite"
|
|
# Default SQLite database file
|
|
DEFAULT_DB="$SCRIPT_DIR/jilo.db"
|
|
# Default MySQL/MariaDB configuration
|
|
DEFAULT_MYSQL_HOST="localhost"
|
|
DEFAULT_MYSQL_USER="jilo"
|
|
DEFAULT_MYSQL_PASS="jilopass"
|
|
DEFAULT_MYSQL_DB="jilo_db"
|
|
|
|
# Load configurations from the config file if it exists
|
|
if [[ -f "$CONFIG_FILE" ]]; then
|
|
source "$CONFIG_FILE"
|
|
fi
|
|
|
|
# use default values if not overriden by config file
|
|
JVB_LOGFILE=${JVB_LOGFILE:-$DEFAULT_JVB_LOGFILE}
|
|
JVB_PROCESS=${JVB_PROCESS:-$DEFAULT_JVB_PROCESS}
|
|
JICOFO_LOGFILE=${JICOFO_LOGFILE:-$DEFAULT_JICOFO_LOGFILE}
|
|
JICOFO_PROCESS=${JICOFO_PROCESS:-$DEFAULT_JICOFO_PROCESS}
|
|
DB=${DB:-$DEFAULT_DB}
|
|
DB_TYPE=${DB_TYPE:-$DEFAULT_DB_TYPE}
|
|
MYSQL_HOST=${MYSQL_HOST:-$DEFAULT_MYSQL_HOST}
|
|
MYSQL_USER=${MYSQL_USER:-$DEFAULT_MYSQL_USER}
|
|
MYSQL_PASS=${MYSQL_PASS:-$DEFAULT_MYSQL_PASS}
|
|
MYSQL_DB=${MYSQL_DB:-$DEFAULT_MYSQL_DB}
|
|
|
|
###
|
|
|
|
# DB queries
|
|
db_get_state_template="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = '%s';"
|
|
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = '%s';"
|
|
|
|
db_get_conference="SELECT * FROM conferences WHERE conference_id = '%s';"
|
|
db_insert_conferences_template="INSERT INTO conferences (jitsi_component, conference_id, conference_name, conference_host) VALUES ('%s', '%s', '%s', '%s');"
|
|
db_insert_conference_event_template="INSERT INTO conference_events (jitsi_component, loglevel, time, conference_id, conference_event, conference_param) VALUES ('%s', '%s', '%s', '%s', '%s', '%s');"
|
|
db_update_conference_id_template="UPDATE conferences SET conference_id='%s' WHERE conference_name = '%s' AND jitsi_component = '%s';"
|
|
## FIXME need a way to update conference_id in Jicofo room creation; conference_name is not unique, need a better way
|
|
db_update_conference_events_id_template="UPDATE conference_events SET conference_id='%s' WHERE conference_name = '%s' AND jitsi_component = '%s';"
|
|
|
|
db_get_participant="SELECT * FROM participants WHERE endpoint_id = '%s';"
|
|
db_insert_participants_template="INSERT INTO participants (jitsi_component, endpoint_id, conference_id) VALUES ('%s', '%s', '%s');"
|
|
db_insert_participant_event_template="INSERT INTO participant_events (jitsi_component, loglevel, time, participant_id, event_type, event_param) VALUES ('%s', '%s', '%s', '%s', '%s', '%s');"
|
|
|
|
db_insert_jitsi_component_event_template="INSERT INTO jitsi_components (jitsi_component, loglevel, time, component_id, event_type, event_param) VALUES ('%s', '%s', '%s', '%s', '%s', '%s');"
|
|
|
|
db_drop="
|
|
DROP TABLE IF EXISTS conferences;
|
|
DROP TABLE IF EXISTS conference_events;
|
|
DROP TABLE IF EXISTS participants;
|
|
DROP TABLE IF EXISTS participant_events;
|
|
DROP TABLE IF EXISTS jitsi_components;
|
|
DROP TABLE IF EXISTS state;"
|
|
db_create="CREATE TABLE conferences (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
jitsi_component TEXT NOT NULL,
|
|
conference_id TEXT NOT NULL,
|
|
conference_name TEXT NOT NULL,
|
|
conference_host TEXT NOT NULL
|
|
);
|
|
CREATE TABLE conference_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
jitsi_component TEXT NOT NULL,
|
|
loglevel TEXT,
|
|
time TEXT NOT NULL,
|
|
conference_id INTEGER NOT NULL,
|
|
conference_event TEXT NOT NULL,
|
|
conference_param TEXT,
|
|
FOREIGN KEY (conference_id) REFERENCES conferences(id)
|
|
);
|
|
CREATE TABLE participants (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
jitsi_component TEXT NOT NULL,
|
|
endpoint_id TEXT NOT NULL,
|
|
conference_id INTEGER NOT NULL,
|
|
FOREIGN KEY (conference_id) REFERENCES conferences(id)
|
|
);
|
|
CREATE TABLE participant_events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
jitsi_component TEXT NOT NULL,
|
|
loglevel TEXT,
|
|
time TEXT NOT NULL,
|
|
participant_id INTEGER NOT NULL,
|
|
event_type TEXT NOT NULL,
|
|
event_param TEXT,
|
|
FOREIGN KEY (participant_id) REFERENCES participants(id)
|
|
);
|
|
CREATE TABLE jitsi_components (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
jitsi_component TEXT NOT NULL,
|
|
loglevel TEXT,
|
|
time TEXT NOT NULL,
|
|
component_id TEXT,
|
|
event_type TEXT,
|
|
event_param TEXT
|
|
);
|
|
CREATE TABLE state (
|
|
id INTEGER PRIMARY KEY,
|
|
jitsi_component TEXT,
|
|
time TEXT,
|
|
filename TEXT,
|
|
filetime INTEGER,
|
|
filesize INTEGER,
|
|
position INTEGER CHECK(typeof(position)='integer'),
|
|
inode INTEGER
|
|
);"
|
|
db_init="
|
|
INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JVB', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);
|
|
INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (2, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
|
|
db_flush="
|
|
DELETE FROM conferences;
|
|
DELETE FROM conference_events;
|
|
DELETE FROM participants;
|
|
DELETE FROM participant_events;
|
|
DELETE FROM jitsi_components;
|
|
DELETE FROM state;"
|
|
|
|
help="Usage:
|
|
$0 [OPTION]
|
|
Options:
|
|
--create-db|-d - create the database
|
|
--flush|-f - flush the tables
|
|
--check|-c [-v] - check for new data [verbosely]
|
|
--parse|-p [-v] - parse the logs [verbosely]
|
|
--help|-h - show this help message
|
|
--version|-V - show version"
|
|
|
|
version="JILO Jitsi Logs Observer
|
|
jilo_${VERSION}_${RELEASE_DATE}
|
|
version $VERSION
|
|
released on $RELEASE_DATE"
|
|
|
|
|
|
###
|
|
|
|
# First we check for requirements
|
|
check_requirements() {
|
|
# required programs, anything non-bash - edit as needed
|
|
# deb packages - sqlite3, coreutils(stat,dd)
|
|
local required_programs=("sqlite3" "stat" "dd")
|
|
local requirements_missing=''
|
|
for program in "${required_programs[@]}"; do
|
|
if ! command -v "$program" &> /dev/null; then
|
|
requirements_missing+="$program, "
|
|
fi
|
|
done
|
|
if [[ "$requirements_missing" != '' ]]; then
|
|
requirements_missing=${requirements_missing::-2}
|
|
echo "Error: $requirements_missing - not found. Please install to proceed."
|
|
exit 1
|
|
fi
|
|
}
|
|
check_requirements
|
|
|
|
###
|
|
|
|
# DB functions for Sqlite3 and for MySQL/MariaDB
|
|
|
|
# normalize DB schemas for Sqlite3 and MySQL/MariaDB in order to compare them when needed
|
|
db_normalize_schema() {
|
|
echo "$1" | tr -d '\n' | tr -s ' ' | tr ',' '\n' | tr -d ';' | sort
|
|
}
|
|
|
|
# execute a query and return the result
|
|
db_query() {
|
|
local query=$1
|
|
if [[ "$DB_TYPE" == "sqlite" ]]; then
|
|
sqlite3 "$DB" "$query"
|
|
elif [[ "$DB_TYPE" == "mysql" || "$DB_TYPE" == "mariadb" ]]; then
|
|
mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p "$MYSQL_PASS" -D "$MYSQL_DB" -se "$query"
|
|
else
|
|
echo "Error: unknown database type $DB_TYPE."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Get the last processed state from the database
|
|
get_state() {
|
|
db_get_state=$(printf "$db_get_state_template" "$1")
|
|
db_query "$db_get_state"
|
|
}
|
|
|
|
# Update the state database
|
|
set_state() {
|
|
local filename=$1
|
|
local filetime=$2
|
|
local filesize=$3
|
|
local position=${4:-0}
|
|
local inode=$5
|
|
local jitsi_component=$6
|
|
db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode" "$jitsi_component")
|
|
db_query "$db_set_state"
|
|
}
|
|
|
|
# Check for a conference and add it if needed
|
|
# input - jitsi_component, conference_id, conference_name, conference_host
|
|
db_conference_check() {
|
|
db_get=$(printf "$db_get_conference" "$2")
|
|
existing_conference=$(db_query "$db_get")
|
|
if [ -z "$existing_conference" ]; then
|
|
# add new conference
|
|
db_insert=$(printf "$db_insert_conferences_template" "$1" "$2" "$3" "$4")
|
|
db_query "$db_insert"
|
|
fi
|
|
}
|
|
|
|
# Check for a participant and add it if needed
|
|
# input - jitsi_component, endpoint_id, conference_id
|
|
db_participant_check() {
|
|
db_get=$(printf "$db_get_participant" "$2")
|
|
existing_participant=$(db_query "$db_get")
|
|
if [ -z "$existing_participant" ]; then
|
|
# add new participant
|
|
db_insert=$(printf "$db_insert_participants_template" "$1" "$2" "$3")
|
|
db_query "$db_insert"
|
|
fi
|
|
}
|
|
|
|
|
|
|
|
###
|
|
|
|
# Main parsing funstion
|
|
jitsi_log_parse() {
|
|
|
|
local file=$1
|
|
local start_pos=$2
|
|
new_last_pos="$start_pos"
|
|
|
|
# Get size and position for progress tracking
|
|
local total_size
|
|
total_size=$(stat -c '%s' "$file")
|
|
local processed_lines=0
|
|
local processed_bytes=0
|
|
|
|
# We open the file and start reading from $start_pos bytes
|
|
exec 3<"$file"
|
|
while IFS= read -r line; do
|
|
# save new position (previous plus bytes in current line plus 1 for the new line)
|
|
new_last_pos=$((new_last_pos + ${#line} + 1))
|
|
|
|
# increment progress stats
|
|
processed_lines=$((processed_lines + 1))
|
|
processed_bytes=$((processed_bytes + ${#line} + 1))
|
|
|
|
# show progress if in verbose mode
|
|
if [[ "$verbose" == true ]]; then
|
|
local percent=$((100 * processed_bytes / total_size))
|
|
echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
|
|
fi
|
|
|
|
case $jitsi_component in
|
|
|
|
# We always check if a conference or participant exists, because
|
|
# incomplete old logs may lead to corrupted stats otherwise
|
|
|
|
JVB)
|
|
# conference starting event
|
|
if [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=([a-zA-Z0-9]+)\ conf_name=(.*)@(.*)\ meeting_id=.*\]\ EndpointConnectionStatusMonitor\.start.*:\ Starting\ connection\ status\ monitor ]]; then
|
|
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
conference_id="${BASH_REMATCH[3]}"
|
|
conference_name="${BASH_REMATCH[4]}"
|
|
conference_host="${BASH_REMATCH[5]}"
|
|
|
|
# always check or add the conference
|
|
db_conference_check "$jitsi_component" "$conference_id" "$conference_name" "$conference_host"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "conference created" "")
|
|
db_query "$db_insert"
|
|
|
|
# locate the conference ending event
|
|
elif [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=([a-zA-Z0-9]+)\ .*\ Conference\.expire ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
conference_id="${BASH_REMATCH[3]}"
|
|
|
|
# always check or add the conference
|
|
db_conference_check "$jitsi_component" "$conference_id" "$conference_name" "$conference_host"
|
|
|
|
db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "conference expired" "")
|
|
db_query "$db_insert"
|
|
|
|
# the conference ended, forget about it
|
|
unset "$conference_id" "$conference_name"
|
|
|
|
# locate participant joining event
|
|
elif [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=([a-zA-Z0-9]+)\ .*epId=([a-zA-Z0-9-]+)\ stats_id=([a-zA-Z0-9-]+)\ .*Starting\ the\ Agent\ without\ remote\ candidates ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
conference_id="${BASH_REMATCH[3]}"
|
|
participant_endpoint_id="${BASH_REMATCH[4]}"
|
|
participant_stats_id="${BASH_REMATCH[5]}"
|
|
|
|
# always check or add the participant
|
|
db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "participant joining" "$conference_id")
|
|
db_query "$db_insert"
|
|
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "stats_id" "$participant_stats_id")
|
|
db_query "$db_insert"
|
|
|
|
# locate participant pair selection event
|
|
elif [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=${conference_id}\ .*epId=${participant_endpoint_id}\ stats_id=${participant_stats_id}\ .*Selected\ pair\ for\ stream\ .*([0-9.]+):10000/udp/srflx\ \-\>\ ([0-9.]+):[0-9]+/udp/prflx ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
participant_IP="${BASH_REMATCH[4]}"
|
|
|
|
# always check or add the participant
|
|
db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "pair selected" "$participant_IP")
|
|
db_query "$db_insert"
|
|
|
|
# locate participant leaving event
|
|
elif [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=${conference_id}\ .*epId=${participant_endpoint_id}\ stats_id=${participant_stats_id}\]\ Endpoint\.expire.*:\ Expired\. ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
|
|
# always check or add the participant
|
|
db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "participant leaving" "$conference_id")
|
|
db_query "$db_insert"
|
|
|
|
# the participant left, forget about him
|
|
unset "$participant_endpoint_id" "$participant_stats_id" "$participant_IP"
|
|
|
|
fi
|
|
;;
|
|
|
|
JICOFO)
|
|
|
|
# jicofo starting
|
|
if [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*Main\.main.*:\ Starting\ Jicofo\. ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
## FIXME a way to add some jicofo id and/or parameter
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "" "jicofo starting" "")
|
|
db_query "$db_insert"
|
|
|
|
# jicofo registered to xmpp
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[xmpp_connection=client\]\ XmppProvider\$connectionListener\$1\.authenticated.*:\ Registered\. ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "" "jicofo xmpp registered" "")
|
|
db_query "$db_insert"
|
|
|
|
# jicofo started
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\JicofoServices\.\<init\>.*\ Registering\ GlobalMetrics\ periodic\ updates\. ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "" "jicofo started" "")
|
|
db_query "$db_insert"
|
|
|
|
# bridge added
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\BridgeSelector\.addJvbAddress.*:\ Added\ new\ videobridge:\ Bridge\[jid=.*@.*\/(.*),\ version=(.*),\ .*\ region=(.*), ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
jvb_id="${BASH_REMATCH[3]}"
|
|
jvb_version="${BASH_REMATCH[4]}"
|
|
jvb_region="${BASH_REMATCH[5]}"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb added" "")
|
|
db_query "$db_insert"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb version" "$jvb_version")
|
|
db_query "$db_insert"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb region" "$jvb_region")
|
|
db_query "$db_insert"
|
|
|
|
# bridge removed
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\BridgeSelector\.removeJvbAddress.*:\ Removing\ JVB:\ .*@.*\/(.*) ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
jvb_id="${BASH_REMATCH[3]}"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb removed" "")
|
|
db_query "$db_insert"
|
|
|
|
# bridge lost (just in case the removal was not detected)
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\BridgeSelector\.removeJvbAddress.*:\ Lost\ a\ bridge:\ .*@.*\/(.*) ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
jvb_id="${BASH_REMATCH[3]}"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb lost" "")
|
|
db_query "$db_insert"
|
|
|
|
# bridge healthcheck scheduled
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\JvbDoctor\.bridgeAdded.*:\ Scheduled\ health-check\ task\ for:\ Bridge\[jid=.*@.*\/(.*),\ version=(.*),\ .*\ region=(.*), ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
jvb_id="${BASH_REMATCH[3]}"
|
|
jvb_version="${BASH_REMATCH[4]}"
|
|
jvb_region="${BASH_REMATCH[5]}"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb health-check scheduled" "")
|
|
db_query "$db_insert"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb version" "$jvb_version")
|
|
db_query "$db_insert"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb region" "$jvb_region")
|
|
db_query "$db_insert"
|
|
|
|
# bridge healthcheck stopped
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\JvbDoctor\.bridgeRemoved.*:\ Stopping\ health-check\ task\ for:\ Bridge\[jid=.*@.*\/(.*),\ version=(.*),\ .*\ region=(.*), ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
jvb_id="${BASH_REMATCH[3]}"
|
|
jvb_version="${BASH_REMATCH[4]}"
|
|
jvb_region="${BASH_REMATCH[5]}"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb health-check stopped" "")
|
|
db_query "$db_insert"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb version" "$jvb_version")
|
|
db_query "$db_insert"
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$jvb_id" "jvb region" "$jvb_region")
|
|
db_query "$db_insert"
|
|
|
|
# WARNING no opertional bridges
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*BridgeSelector\.selectBridge.*\ There\ are\ no\ operational\ bridges\. ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "" "no operational bridges" "")
|
|
db_query "$db_insert"
|
|
|
|
# ERROR no bridge available
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+).*:\ Can\ not\ invite\ participant,\ no\ bridge\ available\. ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
conference_id="${BASH_REMATCH[4]}"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_jitsi_component_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "no bridge available" "")
|
|
db_query "$db_insert"
|
|
|
|
# locate conference starting event
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@(.*)\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
conference_id="0" # FIXME here we still don't have the jicofo room ID
|
|
conference_name="${BASH_REMATCH[3]}"
|
|
conference_host="${BASH_REMATCH[4]}"
|
|
|
|
# always check or add the conference
|
|
db_conference_check "$jitsi_component" "$conference_id" "$conference_name" "$conference_host"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "conference created" "")
|
|
db_query "$db_insert"
|
|
|
|
# locate participant joining event
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ .*\.onMemberJoined.*:\ Member\ joined:([a-zA-Z0-9]+)\ stats-id=([a-zA-Z0-9-]+) ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
conference_name="${BASH_REMATCH[3]}"
|
|
conference_id="${BASH_REMATCH[4]}"
|
|
participant_endpoint_id="${BASH_REMATCH[5]}"
|
|
participant_stats_id="${BASH_REMATCH[6]}"
|
|
|
|
# now we have conf ID update conference
|
|
db_update=$(printf "$db_update_conference_id_template" "$conference_id" "$conference_name" "$jitsi_component")
|
|
db_query "$db_update"
|
|
## FIXME no way to match conference_id here to update it
|
|
# db_update=$(printf "$db_update_conference_events_id_template" "$conference_id" "$conference_name" "$jitsi_component")
|
|
# db_query "$db_update"
|
|
|
|
# always check or add the participant
|
|
db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "participant joining" "$conference_id")
|
|
db_query "$db_insert"
|
|
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "stats_id" "$participant_stats_id")
|
|
db_query "$db_insert"
|
|
|
|
# locate the bridge selection event(s)
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ ColibriV2SessionManager.allocate.*:\ Selected\ (.*),\ session\ exists:\ true ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
# conference_name="${BASH_REMATCH[3]}"
|
|
conference_id="${BASH_REMATCH[4]}"
|
|
bridge_selected="${BASH_REMATCH[5]}"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "bridge selected" "$bridge_selected")
|
|
db_query "$db_insert"
|
|
|
|
# locate participant leaving event
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ .*\.removeParticipant#.*:\ Removing\ ([a-zA-Z0-9]+) ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
# conference_name="${BASH_REMATCH[3]}"
|
|
conference_id="${BASH_REMATCH[4]}"
|
|
participant_endpoint_id="${BASH_REMATCH[5]}"
|
|
|
|
# always check or add the participant
|
|
db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "participant leaving" "$conference_id")
|
|
db_query "$db_insert"
|
|
|
|
# locate the corresponding conference ending event
|
|
elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then
|
|
event_time="${BASH_REMATCH[1]}"
|
|
loglevel="${BASH_REMATCH[2]}"
|
|
# conference_name="${BASH_REMATCH[3]}"
|
|
conference_id="${BASH_REMATCH[4]}"
|
|
|
|
# always check or add the conference
|
|
db_conference_check "$jitsi_component" "$conference_id" "$conference_name" "$conference_host"
|
|
|
|
# add the event details
|
|
db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "conference stopped" "")
|
|
db_query "$db_insert"
|
|
|
|
fi
|
|
;;
|
|
|
|
esac
|
|
|
|
# We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
|
|
done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
|
|
|
|
# Close the file descriptor
|
|
exec 3<&-
|
|
}
|
|
|
|
|
|
### FIXME - this is not currently used
|
|
# check if and which process is running
|
|
is_process_running() {
|
|
pgrep -f "$1" >/dev/null 2>&1
|
|
}
|
|
|
|
### Commandline options
|
|
|
|
verbose=false
|
|
|
|
# Expand combined multiple short options
|
|
expand_options() {
|
|
local arg="$1"
|
|
local expanded=""
|
|
local i
|
|
for ((i=1; i<${#arg}; i++)); do
|
|
expanded="${expanded} -${arg:i:1}"
|
|
done
|
|
echo "$expanded"
|
|
}
|
|
|
|
# We try to distinguish short ("-o") and long ("--option") options
|
|
args=()
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
# only one dash, could be short option
|
|
-[!-]?*)
|
|
args+=("$(expand_options "$1")")
|
|
;;
|
|
# all other cases, including long option
|
|
*)
|
|
args+=("$1")
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# switch between the options
|
|
for arg in "${args[@]}"; do
|
|
case "$arg" in
|
|
-d | --create-db)
|
|
cmd="--create-db"
|
|
;;
|
|
-f | --flush)
|
|
cmd="--flush"
|
|
;;
|
|
-c | --check)
|
|
cmd="--check"
|
|
;;
|
|
-p | --parse)
|
|
cmd="--parse"
|
|
;;
|
|
-v | --verbose)
|
|
verbose=true
|
|
;;
|
|
-h | --help)
|
|
echo -e "$help"
|
|
exit 0
|
|
;;
|
|
-V | --version)
|
|
echo -e "$version"
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Invalid option: -$OPTARG" >&2
|
|
echo -e "$help"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
case "$cmd" in
|
|
|
|
--create-db)
|
|
db_query "$db_drop"
|
|
db_query "$db_create"
|
|
db_query "$db_init"
|
|
echo "Database created."
|
|
exit 0
|
|
;;
|
|
|
|
--flush)
|
|
db_query "$db_flush"
|
|
db_query "$db_init"
|
|
echo "Tables flushed."
|
|
exit 0
|
|
;;
|
|
|
|
--check)
|
|
|
|
# database checks
|
|
if [[ "$DB_TYPE" == "sqlite" ]]; then
|
|
|
|
# First check if database exists
|
|
if [[ ! -f "$DB" ]]; then
|
|
echo "Database not found. If it's a fresh install, please install the database first."
|
|
exit 1
|
|
fi
|
|
|
|
# get current and expected db schemas in comparable format
|
|
#current_db_schema=$(sqlite3 "$DB" .schema)
|
|
current_db_schema=$(sqlite3 "$DB" "SELECT sql FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence';")
|
|
current_db_schema_normalized=$(db_normalize_schema "$current_db_schema")
|
|
expected_db_schema_normalized=$(db_normalize_schema "$db_create")
|
|
|
|
# compare the DB schema to the expected one
|
|
if [[ "$current_db_schema_normalized" != "$expected_db_schema_normalized" ]]; then
|
|
echo "The database doesn't match the expected schema. Please check it, and if needed, reinstall it."
|
|
exit 1
|
|
fi
|
|
|
|
elif [[ "$DB_TYPE" == "mysql" || "$DB_TYPE" == "mariadb" ]]; then
|
|
|
|
# First check if database exists
|
|
if ! mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p "$MYSQL_PASS" -e "USE $MYSQL_DB"; then
|
|
echo "Database not found. If it's a fresh install, please install the database first."
|
|
exit 1
|
|
fi
|
|
|
|
# Get the list of tables, omiting the 'show tables' header
|
|
tables=()
|
|
while IFS= read -r line; do
|
|
tables+=("$line")
|
|
done < <(mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p "$MYSQL_PASS" -D "$MYSQL_DB" -e "SHOW TABLES;" | tail -n +2)
|
|
|
|
# get current and expected db schemas in comparable format
|
|
current_db_schema=''
|
|
for table in "${tables[@]}"; do
|
|
create_table_string=$(mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p "$MYSQL_PASS" -D "$MYSQL_DB" -e "SHOW CREATE TABLE $table\G" | grep -v "Table" | grep -v "Create Table")
|
|
create_table_string="${create_table_string#"${create_table_string%%[^[:space:]]*}"}" # remove leading spaces"
|
|
current_db_schema+="$create_table_string"
|
|
done
|
|
current_db_schema_normalized=$(db_normalize_schema "$current_db_schema")
|
|
expected_db_schema_normalized=$(db_normalize_schema "$db_create")
|
|
|
|
# compare the DB schema to the expected one
|
|
if [[ "$current_db_schema_normalized" != "$expected_db_schema_normalized" ]]; then
|
|
echo "The database doesn't match the expected schema. Please check it, and if needed, reinstall it."
|
|
exit 1
|
|
fi
|
|
|
|
else
|
|
echo "Error: unknown database type $DB_TYPE."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if log files exist
|
|
jvb_found=false
|
|
jicofo_found=false
|
|
|
|
if [[ -f "$JVB_LOGFILE" ]]; then
|
|
jvb_found=true
|
|
jitsi_components+=('JVB')
|
|
fi
|
|
|
|
if [[ -f "$JICOFO_LOGFILE" ]]; then
|
|
jicofo_found=true
|
|
jitsi_components+=('JICOFO')
|
|
fi
|
|
|
|
# if no logs present, exit
|
|
if [[ "$jvb_found" == false && "$jicofo_found" == false ]]; then
|
|
echo "Neither \"$JVB_PROCESS\" ($JVB_LOGFILE) nor \"$JICOFO_PROCESS\" ($JICOFO_LOGFILE) log files are found."
|
|
exit 1
|
|
else
|
|
|
|
# otherwise loop through the found components
|
|
for jitsi_component in "${jitsi_components[@]}"; do
|
|
|
|
# Retrieve last log file and position inside it
|
|
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< "$(get_state "$jitsi_component")"
|
|
|
|
# Initialize logfile vars
|
|
LOGFILE=$(eval "echo \$${jitsi_component}_LOGFILE")
|
|
ROTATED_LOGFILE="$LOGFILE.1"
|
|
current_inode=$(stat -c '%i' "$LOGFILE")
|
|
current_filetime=$(stat -c '%Y' "$LOGFILE")
|
|
current_size=$(stat -c '%s' "$LOGFILE")
|
|
|
|
if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then
|
|
echo "It looks like a fresh install. You can now run log parsing."
|
|
exit 0
|
|
fi
|
|
|
|
echo "File: $last_file"
|
|
# verbose report
|
|
if [[ $verbose == true ]]; then
|
|
echo -e "Last filetime:\t$last_filetime" | expand -t 30
|
|
echo -e "Current filetime:\t$current_filetime" | expand -t 30
|
|
echo -e "Last inode:\t$last_inode" | expand -t 30
|
|
echo -e "Current inode:\t$current_inode" | expand -t 30
|
|
echo -e "Last size:\t$last_size" | expand -t 30
|
|
echo -e "Current size:\t$current_size" | expand -t 30
|
|
echo -e "Last processed position:\t$last_pos" | expand -t 30
|
|
fi
|
|
|
|
if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
|
|
echo -n "Log file has rotated. "
|
|
else
|
|
echo -n "Log file has not rotated. "
|
|
fi
|
|
|
|
if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then
|
|
echo -e "New lines have been added to the log.\n"
|
|
else
|
|
echo -e "No new lines in the log.\n"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
exit 0
|
|
;;
|
|
|
|
--parse)
|
|
|
|
# Check if log files exist
|
|
jvb_found=false
|
|
jicofo_found=false
|
|
|
|
if [[ -f "$JVB_LOGFILE" ]]; then
|
|
jvb_found=true
|
|
jitsi_components+=('JVB')
|
|
fi
|
|
|
|
if [[ -f "$JICOFO_LOGFILE" ]]; then
|
|
jicofo_found=true
|
|
jitsi_components+=('JICOFO')
|
|
fi
|
|
|
|
# if no logs present, exit
|
|
if [[ "$jvb_found" == false && "$jicofo_found" == false ]]; then
|
|
echo "Neither \"$JVB_PROCESS\" ($JVB_LOGFILE) nor \"$JICOFO_PROCESS\" ($JICOFO_LOGFILE) log files are found."
|
|
exit 1
|
|
else
|
|
|
|
# otherwise loop through the found components
|
|
for jitsi_component in "${jitsi_components[@]}"; do
|
|
|
|
# Retrieve last log file and position inside it
|
|
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< "$(get_state "$jitsi_component")"
|
|
|
|
# Initialize logfile vars
|
|
LOGFILE=$(eval "echo \$${jitsi_component}_LOGFILE")
|
|
ROTATED_LOGFILE="$LOGFILE.1"
|
|
last_pos=${last_pos:-0}
|
|
current_inode=$(stat -c '%i' "$LOGFILE")
|
|
current_filetime=$(stat -c '%Y' "$LOGFILE")
|
|
current_size=$(stat -c '%s' "$LOGFILE")
|
|
|
|
# Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate)
|
|
# parse the rotated log file
|
|
if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
|
|
echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE"
|
|
jitsi_log_parse "$ROTATED_LOGFILE" 0 "$verbose"
|
|
last_file="$ROTATED_LOGFILE"
|
|
last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
|
|
last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
|
|
set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode" "$jitsi_component"
|
|
fi
|
|
|
|
# parse the current log file
|
|
echo "Processing the current log file: $LOGFILE"
|
|
jitsi_log_parse "$LOGFILE" "$last_pos" "$verbose"
|
|
|
|
if [[ "$verbose" == true ]]; then
|
|
if [[ "$new_last_pos" == "$last_pos" ]]; then
|
|
echo "The file has not changed, no new data inserted."
|
|
else
|
|
echo -e "\nNew last position after parsing: $new_last_pos"
|
|
fi
|
|
fi
|
|
|
|
# update the state in db
|
|
set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode" "$jitsi_component"
|
|
|
|
done
|
|
|
|
fi
|
|
|
|
echo "Data import finished."
|
|
exit 0
|
|
;;
|
|
|
|
*)
|
|
echo -e "$help"
|
|
exit 1
|
|
;;
|
|
esac
|