New version, uses DB instead of a statefile.

main
Yasen Pramatarov 2024-05-22 19:58:08 +03:00
parent 16356cd3da
commit 032bd1dc8a
2 changed files with 92 additions and 38 deletions

View File

@ -2,59 +2,104 @@
#grep -e 'Videobridge.createConference\|Conference.expire' jvb.log > 11 #grep -e 'Videobridge.createConference\|Conference.expire' jvb.log > 11
LOGFILE="/var/log/jitsi/jvb.log" ### Variables
STATEFILE="/srv/jitsi/bin/jitsi-stats.state"
DB="/srv/jitsi/data/jitsi-stats.db"
if [[ -f "$STATEFILE" ]]; then # log files
last_processed=$(cat "$STATEFILE") LOGFILE="../../jvb.log"
else ROTATED_LOGFILE="../../jvb.log.1"
last_processed="1970-01-01 00:00:00.000" # SQLite database file
fi DB="../../jitsi-stats.db"
###
# SQLite queries
db_get_state='SELECT filename, filetime, position, inode FROM state WHERE id = 1;'
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', position='%s', inode='%s' WHERE id = 1;"
db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');"
# Get the last processed state from the database
get_state() {
sqlite3 "$DB" "$db_get_state"
}
# Update the state database
set_state() {
local filename=$1
local filetime=$2
local position=${3:-0}
local inode=$4
db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$position" "$inode")
sqlite3 "$DB" "$db_set_state"
}
# Retrieve last log file and position inside it
IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state)
# Initialize logfile vars
last_pos=${last_pos:-0}
current_inode=$(stat -c '%i' "$LOGFILE")
current_filetime=$(stat -c '%Y' "$LOGFILE")
# Local assoc array for conference events tracking
declare -A start_times declare -A start_times
declare -A conference_names
jvb_log_parse() { jvb_log_parse() {
local new_last_processed="$last_processed" local file=$1
local start_pos=$2
local new_last_pos="$start_pos"
# We open the file and start reading from $start_pos bytes
exec 3<"$file"
while IFS= read -r line; do while IFS= read -r line; do
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9-]+) ]]; then # save new position (previous plus bytes in current line plus 1 for the new line)
new_last_pos=$((new_last_pos + ${#line} + 1))
# locate conference starting event
if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
timestamp="${BASH_REMATCH[1]}" timestamp="${BASH_REMATCH[1]}"
conferenceId="${BASH_REMATCH[2]}" conferenceId="${BASH_REMATCH[2]}"
if [[ "$timestamp" > "$last_processed" ]]; then
start_times["$conferenceId"]="$timestamp" start_times["$conferenceId"]="$timestamp"
if [[ "$timestamp" > "$new_last_processed" ]]; then
new_last_processed="$timestamp"
fi
fi
elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9-]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then # locate the corresponding conference ending event
timestamp="${BASH_REMATCH[1]}" elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then
end_time="${BASH_REMATCH[1]}"
conferenceId="${BASH_REMATCH[2]}" conferenceId="${BASH_REMATCH[2]}"
conferenceName="${BASH_REMATCH[3]}" conferenceName="${BASH_REMATCH[3]}"
if [[ "$timestamp" > "$last_processed" ]]; then
start_time="${start_times["$conferenceId"]}" start_time="${start_times["$conferenceId"]}"
conference_names["$conferenceId"]="$conferenceName"
if [[ -n "$start_time" ]]; then if [[ -n "$start_time" ]]; then
echo "$conferenceName $eventId $start_time $timestamp" db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
sqlite3 "$DB" <<EOF sqlite3 "$DB" "$db_insert"
INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('$conferenceName', '$conferenceId', '$start_time', '$timestamp');
EOF
unset start_times["$conferenceId"] unset start_times["$conferenceId"]
unset conference_names["$conferenceId"]
fi
if [[ "$fimestamp" > "$new_last_processed" ]]; then
new_last_processed="$timestamp"
fi
fi fi
fi fi
# We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
done < "$LOGFILE" # Close the file descriptor and return the new position value
exec 3<&-
echo "$new_last_processed" > "$STATEFILE" echo "$new_last_pos"
} }
jvb_log_parse # parse the rotated log file
if [[ "$last_file" == "$LOGFILE" && -f "$ROTATED_LOGFILE" ]]; then
echo "Processing the rotated log file: $ROTATED_LOGFILE"
last_pos=$(jvb_log_parse "$ROTATED_LOGFILE" 0)
last_file="$ROTATED_LOGFILE"
last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
set_state "$last_file" "$last_filetime" "$last_pos" "$last_inode"
fi
# parse the current log file
echo "Processing the current log file: $LOGFILE"
new_last_pos=$(jvb_log_parse "$LOGFILE" "$last_pos")
# DEBUG
echo "New last position after parsing: $new_last_pos"
# update the state in db
set_state "$LOGFILE" "$current_filetime" "$new_last_pos" "$current_inode"
echo "Data import finished."

View File

@ -5,3 +5,12 @@ CREATE TABLE IF NOT EXISTS conferences (
start TEXT, start TEXT,
end TEXT end TEXT
); );
CREATE TABLE IF NOT EXISTS state (
id INTEGER PRIMARY_KEY,
time TEXT,
filename TEXT,
filetime INTEGER,
position INTEGER CHECK(typeof(position)='integer'),
inode INTEGER
);
INSERT OR IGNORE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);