Major SQL code refactoring and respective parsing regexps.

main
Yasen Pramatarov 2024-06-11 14:08:06 +03:00
parent 7f14827fba
commit 1ab4fcade0
1 changed files with 168 additions and 95 deletions

263
jilo
View File

@ -50,10 +50,17 @@ MYSQL_DB=${MYSQL_DB:-$DEFAULT_MYSQL_DB}
# DB queries # DB queries
db_get_state_template="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = '%s';" db_get_state_template="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = '%s';"
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = '%s';" db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = '%s';"
db_get_conference="SELECT * FROM conferences WHERE conference_id = '%s';"
db_insert_conferences_template="INSERT INTO conferences (jitsi_component, conference_id, conference_name, conference_host) VALUES ('%s', '%s', '%s', '%s');" db_insert_conferences_template="INSERT INTO conferences (jitsi_component, conference_id, conference_name, conference_host) VALUES ('%s', '%s', '%s', '%s');"
db_insert_conference_event_template="INSERT INTO conference_events (jitsi_component, time, conference_id, conference_event, conference_param) VALUES ('%s', '%s', '%s', '%s', '%s');" db_insert_conference_event_template="INSERT INTO conference_events (jitsi_component, loglevel, time, conference_id, conference_event, conference_param) VALUES ('%s', '%s', '%s', '%s', '%s', '%s');"
db_update_conference_id_template="UPDATE conferences SET conference_id='%s' WHERE conference_name = '%s' AND jitsi_component = '%s';"
## FIXME need a way to update conference_id in Jicofo room creation; conference_name is not unique, need a better way
db_update_conference_events_id_template="UPDATE conference_events SET conference_id='%s' WHERE conference_name = '%s' AND jitsi_component = '%s';"
db_get_participant="SELECT * FROM participants WHERE endpoint_id = '%s';"
db_insert_participants_template="INSERT INTO participants (jitsi_component, endpoint_id, conference_id) VALUES ('%s', '%s', '%s');" db_insert_participants_template="INSERT INTO participants (jitsi_component, endpoint_id, conference_id) VALUES ('%s', '%s', '%s');"
db_insert_participant_event_template="INSERT INTO participants (jitsi_component, time, participant_id, event_type, event_param) VALUES ('%s', '%s', '%s', '%s', '%s');" db_insert_participant_event_template="INSERT INTO participant_events (jitsi_component, loglevel, time, participant_id, event_type, event_param) VALUES ('%s', '%s', '%s', '%s', '%s', '%s');"
db_drop=" db_drop="
DROP TABLE IF EXISTS conferences; DROP TABLE IF EXISTS conferences;
@ -71,6 +78,7 @@ db_create="CREATE TABLE conferences (
CREATE TABLE conference_events ( CREATE TABLE conference_events (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
jitsi_component TEXT NOT NULL, jitsi_component TEXT NOT NULL,
loglevel TEXT,
time TEXT NOT NULL, time TEXT NOT NULL,
conference_id INTEGER NOT NULL, conference_id INTEGER NOT NULL,
conference_event TEXT NOT NULL, conference_event TEXT NOT NULL,
@ -87,6 +95,7 @@ CREATE TABLE participants (
CREATE TABLE participant_events ( CREATE TABLE participant_events (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
jitsi_component TEXT NOT NULL, jitsi_component TEXT NOT NULL,
loglevel TEXT,
time TEXT NOT NULL, time TEXT NOT NULL,
participant_id INTEGER NOT NULL, participant_id INTEGER NOT NULL,
event_type TEXT NOT NULL, event_type TEXT NOT NULL,
@ -182,6 +191,32 @@ set_state() {
db_query "$db_set_state" db_query "$db_set_state"
} }
# Check for a conference and add it if needed
# input - jitsi_component, conference_id, conference_name, conference_host
db_conference_check() {
db_get=$(printf "$db_get_conference" "$2")
existing_conference=$(db_query "$db_get")
if [ -z "$existing_conference" ]; then
# add new conference
db_insert=$(printf "$db_insert_conferences_template" "$1" "$2" "$3" "$4")
db_query "$db_insert"
fi
}
# Check for a participant and add it if needed
# input - jitsi_component, endpoint_id, conference_id
db_participant_check() {
db_get=$(printf "$db_get_participant" "$2")
existing_participant=$(db_query "$db_get")
if [ -z "$existing_participant" ]; then
# add new participant
db_insert=$(printf "$db_insert_participants_template" "$1" "$2" "$3")
db_query "$db_insert"
fi
}
### ###
# Main parsing funstion # Main parsing funstion
@ -191,9 +226,6 @@ jitsi_log_parse() {
local start_pos=$2 local start_pos=$2
new_last_pos="$start_pos" new_last_pos="$start_pos"
# Local assoc array for conference events tracking
declare -A start_times
# Get size and position for progress tracking # Get size and position for progress tracking
local total_size local total_size
total_size=$(stat -c '%s' "$file") total_size=$(stat -c '%s' "$file")
@ -218,130 +250,171 @@ jitsi_log_parse() {
case $jitsi_component in case $jitsi_component in
# We always check if a conference or participant exists, because
# incomplete old logs may lead to corrupted stats otherwise
JVB) JVB)
# conference starting event # conference starting event
if [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=([a-zA-Z]+)\ conf_name=(.*)@([a-zA-Z0-9-_.]+)\ meeting_id=.*\]\ EndpointConnectionStatusMonitor\.start.*:\ Starting\ connection\ status\ monitor ]]; then if [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=([a-zA-Z0-9]+)\ conf_name=(.*)@(.*)\ meeting_id=.*\]\ EndpointConnectionStatusMonitor\.start.*:\ Starting\ connection\ status\ monitor ]]; then
timestamp="${BASH_REMATCH[1]}"
event_time="${BASH_REMATCH[1]}"
loglevel="${BASH_REMATCH[2]}" loglevel="${BASH_REMATCH[2]}"
conference_id="${BASH_REMATCH[3]}" conference_id="${BASH_REMATCH[3]}"
conference_name="${BASH_REMATCH[4]}" conference_name="${BASH_REMATCH[4]}"
conference_host="${BASH_REMATCH[5]}" conference_host="${BASH_REMATCH[5]}"
# we check if the conference exists, just in case
db_get=$(printf "$db_get_conference" "$conference_id") # always check or add the conference
existing_conference=$(db_query "$db_get") db_conference_check "$jitsi_component" "$conference_id" "$conference_name" "$conference_host"
# new conference
if [ -z "$existing_conference" ]; then # add the event details
db_insert=$(printf "$db_insert_conferences_template" "$jitsi_component" "$conference_id" "$conference_name" "$conference_host") db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "conference created" "")
db_query "$db_insert" db_query "$db_insert"
# if for any reason it's an existing conference, update details
else
## conference exists add info
fi
# locate participant joining event # locate participant joining event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=${conference_id}\ .*epId=([a-zA-Z0-9-]+)\ stats_id=([a-zA-Z0-9-]+)\ .*Starting\ the\ Agent\ without\ remote\ candidates ]]; then elif [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=([a-zA-Z0-9]+)\ .*epId=([a-zA-Z0-9-]+)\ stats_id=([a-zA-Z0-9-]+)\ .*Starting\ the\ Agent\ without\ remote\ candidates ]]; then
event_time="${BASH_REMATCH[1]}" event_time="${BASH_REMATCH[1]}"
event_type='participant joining' loglevel="${BASH_REMATCH[2]}"
participant_endpoint_id="${BASH_REMATCH[2]}" conference_id="${BASH_REMATCH[3]}"
participant_stats_id="${BASH_REMATCH[3]}" participant_endpoint_id="${BASH_REMATCH[4]}"
if [[ -n "$conference_id" ]]; then participant_stats_id="${BASH_REMATCH[5]}"
db_insert=$(printf "$db_insert_participant_template" "$jitsi_component" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "")
db_query "$db_insert" # always check or add the participant
fi db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
# add the event details
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "participant joining" "$conference_id")
db_query "$db_insert"
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "stats_id" "$participant_stats_id")
db_query "$db_insert"
# locate participant pair selection event # locate participant pair selection event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=${conference_id}\ .*epId=${participant_endpoint_id}\ stats_id=${participant_stats_id}\ .*Selected\ pair\ for\ stream\ .*([0-9.]+):10000/udp/srflx\ \-\>\ ([0-9.]+):[0-9]+/udp/prflx ]]; then elif [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=${conference_id}\ .*epId=${participant_endpoint_id}\ stats_id=${participant_stats_id}\ .*Selected\ pair\ for\ stream\ .*([0-9.]+):10000/udp/srflx\ \-\>\ ([0-9.]+):[0-9]+/udp/prflx ]]; then
event_time="${BASH_REMATCH[1]}" event_time="${BASH_REMATCH[1]}"
event_type='pair selected' loglevel="${BASH_REMATCH[2]}"
participant_IP="${BASH_REMATCH[3]}" participant_IP="${BASH_REMATCH[4]}"
if [[ -n "$conference_id" ]]; then
db_insert=$(printf "$db_insert_participant_template" "$jitsi_component" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "$participant_IP") # always check or add the participant
db_query "$db_insert" db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
fi
# add the event details
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "pair selected" "$participant_IP")
db_query "$db_insert"
# locate participant leaving event # locate participant leaving event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=${conference_id}\ .*epId=${participant_endpoint_id}\ stats_id=${participant_stats_id}\]\ Endpoint\.expire.*:\ Expired\. ]]; then elif [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=${conference_id}\ .*epId=${participant_endpoint_id}\ stats_id=${participant_stats_id}\]\ Endpoint\.expire.*:\ Expired\. ]]; then
event_time="${BASH_REMATCH[1]}" event_time="${BASH_REMATCH[1]}"
event_type='participant leaving' loglevel="${BASH_REMATCH[2]}"
if [[ -n "$conference_id" ]]; then
db_insert=$(printf "$db_insert_participant_template" "$jitsi_component" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "$participant_IP")
db_query "$db_insert"
# the participant left, forget about him
unset "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "$participant_IP"
fi
# locate the corresponding conference ending event # always check or add the participant
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
end_time="${BASH_REMATCH[1]}"
conference_id="${BASH_REMATCH[2]}" # add the event details
conference_name="${BASH_REMATCH[3]}" db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "participant leaving" "$conference_id")
start_time="${start_times["$conference_id"]}" db_query "$db_insert"
# the participant left, forget about him
unset "$participant_endpoint_id" "$participant_stats_id" "$participant_IP"
# locate the conference ending event
elif [[ "$line" =~ ${jitsi_component}\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[confId=([a-zA-Z0-9]+)\ .*\ Conference\.expire ]]; then
event_time="${BASH_REMATCH[1]}"
loglevel="${BASH_REMATCH[2]}"
conference_id="${BASH_REMATCH[3]}"
# always check or add the conference
db_conference_check "$jitsi_component" "$conference_id" "$conference_name" "$conference_host"
db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "conference expired" "")
db_query "$db_insert"
# the conference ended, forget about it
unset "$conference_id" "$conference_name"
if [[ -n "$start_time" ]]; then
db_insert=$(printf "$db_insert_conference_template" "$jitsi_component" "$conference_name" "$conference_id" "$start_time" "$end_time")
db_query "$db_insert"
# the conference ended, forget about it
unset "start_times[$conference_id]" "$start_time" "$end_time" "$conference_id" "$conference_name"
fi
fi fi
;; ;;
JICOFO) JICOFO)
# locate conference starting event # locate conference starting event
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then if [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@(.*)\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then
timestamp="${BASH_REMATCH[1]}" event_time="${BASH_REMATCH[1]}"
conference_name="${BASH_REMATCH[2]}" loglevel="${BASH_REMATCH[2]}"
start_times["$conference_name"]="$timestamp" conference_id="0" # FIXME here we still don't have the jicofo room ID
conference_name="${BASH_REMATCH[3]}"
conference_host="${BASH_REMATCH[4]}"
# always check or add the conference
db_conference_check "$jitsi_component" "$conference_id" "$conference_name" "$conference_host"
# add the event details
db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "conference created" "")
db_query "$db_insert"
# locate participant joining event # locate participant joining event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ .*\.onMemberJoined.*:\ Member\ joined:([a-zA-Z0-9]+)\ stats-id=([a-zA-Z0-9-]+) ]]; then elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ .*\.onMemberJoined.*:\ Member\ joined:([a-zA-Z0-9]+)\ stats-id=([a-zA-Z0-9-]+) ]]; then
event_time="${BASH_REMATCH[1]}" event_time="${BASH_REMATCH[1]}"
event_type='participant joining' loglevel="${BASH_REMATCH[2]}"
conference_name="${BASH_REMATCH[2]}" conference_name="${BASH_REMATCH[3]}"
conference_id="${BASH_REMATCH[3]}" conference_id="${BASH_REMATCH[4]}"
participant_endpoint_id="${BASH_REMATCH[4]}" participant_endpoint_id="${BASH_REMATCH[5]}"
participant_stats_id="${BASH_REMATCH[5]}" participant_stats_id="${BASH_REMATCH[6]}"
if [[ -n "$conference_id" ]]; then
db_insert=$(printf "$db_insert_participant_template" "$jitsi_component" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "") # now we have conf ID update conference
db_query "$db_insert" db_update=$(printf "$db_update_conference_id_template" "$conference_id" "$conference_name" "$jitsi_component")
fi db_query "$db_update"
## FIXME no way to match conference_id here to update it
# db_update=$(printf "$db_update_conference_events_id_template" "$conference_id" "$conference_name" "$jitsi_component")
# db_query "$db_update"
# always check or add the participant
db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
# add the event details
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "participant joining" "$conference_id")
db_query "$db_insert"
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "stats_id" "$participant_stats_id")
db_query "$db_insert"
# locate the bridge selection event(s) # locate the bridge selection event(s)
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*BridgeSelectionStrategy.select#.*:\ Selected.*\[jid=[a-z]@.*\/(.*),\ ]]; then elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ ColibriV2SessionManager.allocate.*:\ Selected\ (.*),\ session\ exists:\ true ]]; then
event_time="${BASH_REMATCH[1]}" event_time="${BASH_REMATCH[1]}"
event_type='bridge selected' loglevel="${BASH_REMATCH[2]}"
## FIXME, no data conference_name="${BASH_REMATCH[2]}" # conference_name="${BASH_REMATCH[3]}"
## conference_id="${BASH_REMATCH[3]}" conference_id="${BASH_REMATCH[4]}"
## participant_endpoint_id="${BASH_REMATCH[4]}" bridge_selected="${BASH_REMATCH[5]}"
## participant_stats_id="${BASH_REMATCH[5]}"
if [[ -n "$conference_id" ]]; then # add the event details
db_insert=$(printf "$db_insert_conference_template" "$jitsi_component" "$conference_name" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "") db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "bridge selected" "$bridge_selected")
db_query "$db_insert" db_query "$db_insert"
fi
# locate participant leaving event # locate participant leaving event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=${conference_name}@.*\ meeting_id=${conference_id}\]\ .*\.removeParticipant#.*:\ Removing\ ([a-zA-Z0-9]+) ]]; then elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ .*\.removeParticipant#.*:\ Removing\ ([a-zA-Z0-9]+) ]]; then
event_time="${BASH_REMATCH[1]}" event_time="${BASH_REMATCH[1]}"
event_type='participant leaving' loglevel="${BASH_REMATCH[2]}"
participant_endpoint_id="${BASH_REMATCH[2]}" # conference_name="${BASH_REMATCH[3]}"
participant_stats_id="" # FIXME conference_id="${BASH_REMATCH[4]}"
if [[ -n "$conference_id" ]]; then participant_endpoint_id="${BASH_REMATCH[5]}"
db_insert=$(printf "$db_insert_participant_template" "$jitsi_component" "$conference_id" "$event_time" "$event_type" "$participant_endpoint_id" "$participant_stats_id" "")
db_query "$db_insert" # always check or add the participant
fi db_participant_check "$jitsi_component" "$participant_endpoint_id" "$conference_id"
# add the event details
db_insert=$(printf "$db_insert_participant_event_template" "$jitsi_component" "$loglevel" "$event_time" "$participant_endpoint_id" "participant leaving" "$conference_id")
db_query "$db_insert"
# locate the corresponding conference ending event # locate the corresponding conference ending event
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then elif [[ "$line" =~ Jicofo\ ([0-9-]+\ [0-9:.]+)\ ([A-Z]+):.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then
end_time="${BASH_REMATCH[1]}" event_time="${BASH_REMATCH[1]}"
conference_name="${BASH_REMATCH[2]}" loglevel="${BASH_REMATCH[2]}"
conference_id="${BASH_REMATCH[3]}" # conference_name="${BASH_REMATCH[3]}"
start_time="${start_times["$conference_name"]}" conference_id="${BASH_REMATCH[4]}"
# always check or add the conference
db_conference_check "$jitsi_component" "$conference_id" "$conference_name" "$conference_host"
# add the event details
db_insert=$(printf "$db_insert_conference_event_template" "$jitsi_component" "$loglevel" "$event_time" "$conference_id" "conference stopped" "")
db_query "$db_insert"
if [[ -n "$start_time" ]]; then
db_insert=$(printf "$db_insert_conference_template" "$jitsi_component" "$conference_name" "$conference_id" "$start_time" "$end_time")
db_query "$db_insert"
unset "start_times[$conference_name]"
fi
fi fi
;; ;;