Combines JVB and JICOVO checks in one main script.
parent
3b36033be6
commit
10a746ab74
|
|
@ -1,268 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
### Variables
|
||||
|
||||
# log files
|
||||
LOGFILE="./jicofo.log"
|
||||
ROTATED_LOGFILE="./jicofo.log.1"
|
||||
# SQLite database file
|
||||
DB="./jitsi-stats.db"
|
||||
|
||||
###
|
||||
|
||||
# SQLite queries
|
||||
db_get_state="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = 'JICOFO';"
|
||||
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = 'JICOFO';"
|
||||
db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');"
|
||||
db_create="
|
||||
DROP TABLE IF EXISTS conferences;
|
||||
CREATE TABLE conferences (
|
||||
id INTEGER PRIMARY_KEY,
|
||||
conference_name TEXT,
|
||||
conference_id TEXT,
|
||||
start TEXT,
|
||||
end TEXT
|
||||
);
|
||||
DROP TABLE IF EXISTS state;
|
||||
CREATE TABLE state (
|
||||
id INTEGER PRIMARY_KEY,
|
||||
jitsi_component TEXT,
|
||||
time TEXT,
|
||||
filename TEXT,
|
||||
filetime INTEGER,
|
||||
filesize INTEGER,
|
||||
position INTEGER CHECK(typeof(position)='integer'),
|
||||
inode INTEGER
|
||||
);
|
||||
INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
|
||||
db_flush="
|
||||
DELETE FROM conferences;
|
||||
DELETE FROM state;
|
||||
INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
|
||||
|
||||
help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]"
|
||||
|
||||
###
|
||||
|
||||
# First we check for requirements
|
||||
check_requirements() {
|
||||
# required programs, anything non-bash - edit as needed
|
||||
# deb packages - sqlite3, coreutils(stat,dd)
|
||||
local required_programs=("sqlite3" "stat" "dd")
|
||||
local requirements_missing=''
|
||||
for program in "${required_programs[@]}"; do
|
||||
if ! command -v "$program" &> /dev/null; then
|
||||
requirements_missing+="$program, "
|
||||
fi
|
||||
done
|
||||
if [[ "$requirements_missing" != '' ]]; then
|
||||
requirements_missing=${requirements_missing::-2}
|
||||
echo "Error: $requirements_missing - not found. Please install to proceed."
|
||||
fi
|
||||
}
|
||||
check_requirements
|
||||
|
||||
###
|
||||
|
||||
# DB functions
|
||||
|
||||
# Get the last processed state from the database
|
||||
get_state() {
|
||||
sqlite3 "$DB" "$db_get_state"
|
||||
}
|
||||
|
||||
# Update the state database
|
||||
set_state() {
|
||||
local filename=$1
|
||||
local filetime=$2
|
||||
local filesize=$3
|
||||
local position=${4:-0}
|
||||
local inode=$5
|
||||
db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode")
|
||||
sqlite3 "$DB" "$db_set_state"
|
||||
}
|
||||
|
||||
###
|
||||
|
||||
# Main parsing funstion
|
||||
jicofo_log_parse() {
|
||||
|
||||
local file=$1
|
||||
local start_pos=$2
|
||||
new_last_pos="$start_pos"
|
||||
|
||||
# Local assoc array for conference events tracking
|
||||
declare -A start_times
|
||||
|
||||
# Get size and position for progress tracking
|
||||
local total_size=$(stat -c '%s' "$file")
|
||||
local processed_lines=0
|
||||
local procesed_bytes=0
|
||||
|
||||
# We open the file and start reading from $start_pos bytes
|
||||
exec 3<"$file"
|
||||
while IFS= read -r line; do
|
||||
# save new position (previous plus bytes in current line plus 1 for the new line)
|
||||
new_last_pos=$((new_last_pos + ${#line} + 1))
|
||||
|
||||
# increment progress stats
|
||||
processed_lines=$((processed_lines + 1))
|
||||
processed_bytes=$((processed_bytes + ${#line} + 1))
|
||||
|
||||
# show progress if in verbose mode
|
||||
if [[ "$verbose" == true ]]; then
|
||||
local percent=$((100 * processed_bytes / total_size))
|
||||
echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
|
||||
fi
|
||||
|
||||
# locate conference starting event
|
||||
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then
|
||||
timestamp="${BASH_REMATCH[1]}"
|
||||
conferenceName="${BASH_REMATCH[2]}"
|
||||
start_times["$conferenceName"]="$timestamp"
|
||||
|
||||
# locate the corresponding conference ending event
|
||||
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then
|
||||
end_time="${BASH_REMATCH[1]}"
|
||||
conferenceName="${BASH_REMATCH[2]}"
|
||||
conferenceId="${BASH_REMATCH[3]}"
|
||||
start_time="${start_times["$conferenceName"]}"
|
||||
|
||||
if [[ -n "$start_time" ]]; then
|
||||
db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
|
||||
sqlite3 "$DB" "$db_insert"
|
||||
unset start_times["$conferenceName"]
|
||||
fi
|
||||
fi
|
||||
# We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
|
||||
done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
|
||||
|
||||
# Close the file descriptor
|
||||
exec 3<&-
|
||||
}
|
||||
|
||||
###
|
||||
|
||||
# commandline options
|
||||
|
||||
while getopts ":dfcpv" opt; do
|
||||
case $opt in
|
||||
d)
|
||||
cmd="--create-db"
|
||||
;;
|
||||
f)
|
||||
cmd="--flush"
|
||||
;;
|
||||
c)
|
||||
cmd="--check"
|
||||
;;
|
||||
p)
|
||||
cmd="--parse"
|
||||
;;
|
||||
v)
|
||||
verbose=true
|
||||
;;
|
||||
\?)
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
echo -e "$help"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND -1))
|
||||
|
||||
case "$cmd" in
|
||||
|
||||
--create-db)
|
||||
sqlite3 "$DB" "$db_create"
|
||||
echo "Database created."
|
||||
exit 0
|
||||
;;
|
||||
|
||||
--flush)
|
||||
sqlite3 "$DB" "$db_flush"
|
||||
echo "Tables flushed."
|
||||
exit 0
|
||||
;;
|
||||
|
||||
--check)
|
||||
|
||||
# Retrieve last log file and position inside it
|
||||
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)
|
||||
|
||||
# Initialize logfile vars
|
||||
current_inode=$(stat -c '%i' "$LOGFILE")
|
||||
current_filetime=$(stat -c '%Y' "$LOGFILE")
|
||||
current_size=$(stat -c '%s' "$LOGFILE")
|
||||
|
||||
if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then
|
||||
echo "It looks like a fresh install. You can now run log parsing."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# report
|
||||
echo "Last file: $last_file"
|
||||
echo "Last filetime: $last_filetime"
|
||||
echo "Last inode: $last_inode"
|
||||
echo "Last size: $last_size"
|
||||
echo "Last processed position: $last_pos"
|
||||
echo "Current filetime: $current_filetime"
|
||||
echo "Current inode: $current_inode"
|
||||
echo "Current size: $current_size"
|
||||
|
||||
if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
|
||||
echo "Log file has rotated."
|
||||
else
|
||||
echo "Log file has not rotated."
|
||||
fi
|
||||
|
||||
if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then
|
||||
echo "New lines have been added to the log."
|
||||
else
|
||||
echo "No new lines in the log."
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
|
||||
--parse)
|
||||
|
||||
# Retrieve last log file and position inside it
|
||||
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)
|
||||
|
||||
# Initialize logfile vars
|
||||
last_pos=${last_pos:-0}
|
||||
current_inode=$(stat -c '%i' "$LOGFILE")
|
||||
current_filetime=$(stat -c '%Y' "$LOGFILE")
|
||||
current_size=$(stat -c '%s' "$LOGFILE")
|
||||
|
||||
# Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate)
|
||||
# parse the rotated log file
|
||||
if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
|
||||
echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE"
|
||||
jvb_log_parse "$ROTATED_LOGFILE" 0 "$verbose"
|
||||
last_file="$ROTATED_LOGFILE"
|
||||
last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
|
||||
last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
|
||||
set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode"
|
||||
fi
|
||||
|
||||
# parse the current log file
|
||||
echo "Processing the current log file: $LOGFILE"
|
||||
jvb_log_parse "$LOGFILE" "$last_pos" "$verbose"
|
||||
|
||||
if [[ "$verbose" == true ]]; then
|
||||
echo -e "\nNew last position after parsing: $new_last_pos"
|
||||
fi
|
||||
|
||||
# update the state in db
|
||||
set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode"
|
||||
|
||||
echo "Data import finished."
|
||||
exit 0
|
||||
;;
|
||||
|
||||
*)
|
||||
echo -e "$help"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
@ -30,12 +30,13 @@ DB=${DB:-$DEFAULT_DB}
|
|||
# SQLite queries
|
||||
db_get_state_template="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = '%s';"
|
||||
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = '%s';"
|
||||
db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');"
|
||||
db_insert_template="INSERT INTO conferences (jitsi_component, conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s', '%s');"
|
||||
db_drop="
|
||||
DROP TABLE IF EXISTS conferences;
|
||||
DROP TABLE IF EXISTS state;"
|
||||
db_create="CREATE TABLE conferences (
|
||||
id INTEGER PRIMARY_KEY,
|
||||
jitsi_component TEXT,
|
||||
conference_name TEXT,
|
||||
conference_id TEXT,
|
||||
start TEXT,
|
||||
|
|
@ -104,7 +105,7 @@ set_state() {
|
|||
|
||||
###
|
||||
|
||||
# Main parsing funstion
|
||||
# Jitsi Videobridge parsing funstion
|
||||
jvb_log_parse() {
|
||||
|
||||
local file=$1
|
||||
|
|
@ -135,6 +136,9 @@ jvb_log_parse() {
|
|||
echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
|
||||
fi
|
||||
|
||||
case $jitsi_component in
|
||||
|
||||
JVB)
|
||||
# locate conference starting event
|
||||
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
|
||||
timestamp="${BASH_REMATCH[1]}"
|
||||
|
|
@ -149,11 +153,96 @@ jvb_log_parse() {
|
|||
start_time="${start_times["$conferenceId"]}"
|
||||
|
||||
if [[ -n "$start_time" ]]; then
|
||||
db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
|
||||
db_insert=$(printf "$db_insert_template" "$jitsi_component" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
|
||||
sqlite3 "$DB" "$db_insert"
|
||||
unset start_times["$conferenceId"]
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
JICOFO)
|
||||
# locate conference starting event
|
||||
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then
|
||||
timestamp="${BASH_REMATCH[1]}"
|
||||
conferenceName="${BASH_REMATCH[2]}"
|
||||
start_times["$conferenceName"]="$timestamp"
|
||||
|
||||
# locate the corresponding conference ending event
|
||||
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then
|
||||
end_time="${BASH_REMATCH[1]}"
|
||||
conferenceName="${BASH_REMATCH[2]}"
|
||||
conferenceId="${BASH_REMATCH[3]}"
|
||||
start_time="${start_times["$conferenceName"]}"
|
||||
|
||||
if [[ -n "$start_time" ]]; then
|
||||
db_insert=$(printf "$db_insert_template" "$jitsi_component" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
|
||||
sqlite3 "$DB" "$db_insert"
|
||||
unset start_times["$conferenceName"]
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
esac
|
||||
|
||||
# We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
|
||||
done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
|
||||
|
||||
# Close the file descriptor
|
||||
exec 3<&-
|
||||
}
|
||||
|
||||
###
|
||||
|
||||
# Jicofo parsing funstion
|
||||
jicofo_log_parse() {
|
||||
|
||||
local file=$1
|
||||
local start_pos=$2
|
||||
new_last_pos="$start_pos"
|
||||
|
||||
# Local assoc array for conference events tracking
|
||||
declare -A start_times
|
||||
|
||||
# Get size and position for progress tracking
|
||||
local total_size=$(stat -c '%s' "$file")
|
||||
local processed_lines=0
|
||||
local procesed_bytes=0
|
||||
|
||||
# We open the file and start reading from $start_pos bytes
|
||||
exec 3<"$file"
|
||||
while IFS= read -r line; do
|
||||
# save new position (previous plus bytes in current line plus 1 for the new line)
|
||||
new_last_pos=$((new_last_pos + ${#line} + 1))
|
||||
|
||||
# increment progress stats
|
||||
processed_lines=$((processed_lines + 1))
|
||||
processed_bytes=$((processed_bytes + ${#line} + 1))
|
||||
|
||||
# show progress if in verbose mode
|
||||
if [[ "$verbose" == true ]]; then
|
||||
local percent=$((100 * processed_bytes / total_size))
|
||||
echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
|
||||
fi
|
||||
|
||||
# locate conference starting event
|
||||
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then
|
||||
timestamp="${BASH_REMATCH[1]}"
|
||||
conferenceName="${BASH_REMATCH[2]}"
|
||||
start_times["$conferenceName"]="$timestamp"
|
||||
|
||||
# locate the corresponding conference ending event
|
||||
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then
|
||||
end_time="${BASH_REMATCH[1]}"
|
||||
conferenceName="${BASH_REMATCH[2]}"
|
||||
conferenceId="${BASH_REMATCH[3]}"
|
||||
start_time="${start_times["$conferenceName"]}"
|
||||
|
||||
if [[ -n "$start_time" ]]; then
|
||||
db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
|
||||
sqlite3 "$DB" "$db_insert"
|
||||
unset start_times["$conferenceName"]
|
||||
fi
|
||||
fi
|
||||
# We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
|
||||
done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
|
||||
|
||||
|
|
@ -161,7 +250,6 @@ jvb_log_parse() {
|
|||
exec 3<&-
|
||||
}
|
||||
|
||||
###
|
||||
|
||||
### FIXME - this is not currently used
|
||||
# check if and which process is running
|
||||
|
|
@ -253,7 +341,7 @@ case "$cmd" in
|
|||
for jitsi_component in "${jitsi_components[@]}"; do
|
||||
|
||||
# Retrieve last log file and position inside it
|
||||
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)
|
||||
IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state "$jitsi_component")
|
||||
|
||||
# Initialize logfile vars
|
||||
LOGFILE=$(eval "echo \$${jitsi_component}_LOGFILE")
|
||||
|
|
@ -284,9 +372,9 @@ case "$cmd" in
|
|||
fi
|
||||
|
||||
if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then
|
||||
echo "New lines have been added to the log."
|
||||
echo -e "New lines have been added to the log.\n"
|
||||
else
|
||||
echo "No new lines in the log."
|
||||
echo -e "No new lines in the log.\n"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
Loading…
Reference in New Issue