From 032bd1dc8a786374d01967a9021f35463aababba Mon Sep 17 00:00:00 2001 From: Yasen Pramatarov Date: Wed, 22 May 2024 19:58:08 +0300 Subject: [PATCH] New version, uses DB instead of a statefile. --- jvb/jitsi-stats | 121 ++++++++++++++++++++++++++++------------- jvb/jitsi-stats.schema | 9 +++ 2 files changed, 92 insertions(+), 38 deletions(-) diff --git a/jvb/jitsi-stats b/jvb/jitsi-stats index 0159507..ed74b54 100755 --- a/jvb/jitsi-stats +++ b/jvb/jitsi-stats @@ -2,59 +2,104 @@ #grep -e 'Videobridge.createConference\|Conference.expire' jvb.log > 11 -LOGFILE="/var/log/jitsi/jvb.log" -STATEFILE="/srv/jitsi/bin/jitsi-stats.state" -DB="/srv/jitsi/data/jitsi-stats.db" +### Variables -if [[ -f "$STATEFILE" ]]; then - last_processed=$(cat "$STATEFILE") -else - last_processed="1970-01-01 00:00:00.000" -fi +# log files +LOGFILE="../../jvb.log" +ROTATED_LOGFILE="../../jvb.log.1" +# SQLite database file +DB="../../jitsi-stats.db" +### + +# SQLite queries +db_get_state='SELECT filename, filetime, position, inode FROM state WHERE id = 1;' +db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', position='%s', inode='%s' WHERE id = 1;" +db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');" + +# Get the last processed state from the database +get_state() { + sqlite3 "$DB" "$db_get_state" +} + +# Update the state database +set_state() { + local filename=$1 + local filetime=$2 + local position=${3:-0} + local inode=$4 + db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$position" "$inode") + sqlite3 "$DB" "$db_set_state" +} + +# Retrieve last log file and position inside it +IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state) + +# Initialize logfile vars +last_pos=${last_pos:-0} +current_inode=$(stat -c '%i' "$LOGFILE") +current_filetime=$(stat -c '%Y' "$LOGFILE") + +# Local assoc array for conference events tracking declare -A start_times -declare -A conference_names jvb_log_parse() { - local new_last_processed="$last_processed" + local file=$1 + local start_pos=$2 + local new_last_pos="$start_pos" + # We open the file and start reading from $start_pos bytes + exec 3<"$file" while IFS= read -r line; do - if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9-]+) ]]; then + # save new position (previous plus bytes in current line plus 1 for the new line) + new_last_pos=$((new_last_pos + ${#line} + 1)) + + # locate conference starting event + if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then timestamp="${BASH_REMATCH[1]}" conferenceId="${BASH_REMATCH[2]}" - if [[ "$timestamp" > "$last_processed" ]]; then - start_times["$conferenceId"]="$timestamp" - if [[ "$timestamp" > "$new_last_processed" ]]; then - new_last_processed="$timestamp" - fi - fi + start_times["$conferenceId"]="$timestamp" - elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9-]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then - timestamp="${BASH_REMATCH[1]}" + # locate the corresponding conference ending event + elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then + end_time="${BASH_REMATCH[1]}" conferenceId="${BASH_REMATCH[2]}" conferenceName="${BASH_REMATCH[3]}" - if [[ "$timestamp" > "$last_processed" ]]; then - start_time="${start_times["$conferenceId"]}" - conference_names["$conferenceId"]="$conferenceName" - if [[ -n "$start_time" ]]; then - echo "$conferenceName $eventId $start_time $timestamp" - sqlite3 "$DB" < "$new_last_processed" ]]; then - new_last_processed="$timestamp" - fi + start_time="${start_times["$conferenceId"]}" + + if [[ -n "$start_time" ]]; then + db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time") + sqlite3 "$DB" "$db_insert" + unset start_times["$conferenceId"] fi fi + # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value + done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null) - done < "$LOGFILE" - - echo "$new_last_processed" > "$STATEFILE" - + # Close the file descriptor and return the new position value + exec 3<&- + echo "$new_last_pos" } -jvb_log_parse +# parse the rotated log file +if [[ "$last_file" == "$LOGFILE" && -f "$ROTATED_LOGFILE" ]]; then + echo "Processing the rotated log file: $ROTATED_LOGFILE" + last_pos=$(jvb_log_parse "$ROTATED_LOGFILE" 0) + last_file="$ROTATED_LOGFILE" + last_inode=$(stat -c '%i' "$ROTATED_LOGFILE") + last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE") + set_state "$last_file" "$last_filetime" "$last_pos" "$last_inode" +fi + +# parse the current log file +echo "Processing the current log file: $LOGFILE" +new_last_pos=$(jvb_log_parse "$LOGFILE" "$last_pos") + +# DEBUG +echo "New last position after parsing: $new_last_pos" + +# update the state in db +set_state "$LOGFILE" "$current_filetime" "$new_last_pos" "$current_inode" + +echo "Data import finished." diff --git a/jvb/jitsi-stats.schema b/jvb/jitsi-stats.schema index 6d1e60a..caeba5f 100644 --- a/jvb/jitsi-stats.schema +++ b/jvb/jitsi-stats.schema @@ -5,3 +5,12 @@ CREATE TABLE IF NOT EXISTS conferences ( start TEXT, end TEXT ); +CREATE TABLE IF NOT EXISTS state ( + id INTEGER PRIMARY_KEY, + time TEXT, + filename TEXT, + filetime INTEGER, + position INTEGER CHECK(typeof(position)='integer'), + inode INTEGER +); +INSERT OR IGNORE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);