diff --git a/jicofo/jicofo-stats b/jicofo/jicofo-stats new file mode 100755 index 0000000..107bfe8 --- /dev/null +++ b/jicofo/jicofo-stats @@ -0,0 +1,270 @@ +#!/usr/bin/env bash + +### Variables + +# log files +LOGFILE="./jicofo.log" +ROTATED_LOGFILE="./jicofo.log.1" +# SQLite database file +DB="./jitsi-stats.db" + +### + +# SQLite queries +db_get_state="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = 'JICOFO';" +db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = 'JICOFO';" +db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');" +db_create=" +DROP TABLE IF EXISTS conferences; +CREATE TABLE conferences ( + id INTEGER PRIMARY_KEY, + conference_name TEXT, + conference_id TEXT, + start TEXT, + end TEXT +); +DROP TABLE IF EXISTS state; +CREATE TABLE state ( + id INTEGER PRIMARY_KEY, + jitsi_component TEXT, + time TEXT, + filename TEXT, + filetime INTEGER, + filesize INTEGER, + position INTEGER CHECK(typeof(position)='integer'), + inode INTEGER +); +INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);" +db_flush=" +DELETE FROM conferences; +DELETE FROM state; +INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);" + +help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]" + +### + +# First we check for requirements +check_requirements() { + # required programs, anything non-bash - edit as needed + # deb packages - sqlite3, coreutils(stat,dd) + local required_programs=("sqlite3" "stat" "dd") + local requirements_missing='' + for program in "${required_programs[@]}"; do + if ! command -v "$program" &> /dev/null; then + requirements_missing+="$program, " + fi + done + if [[ "$requirements_missing" != '' ]]; then + requirements_missing=${requirements_missing::-2} + echo "Error: $requirements_missing - not found. Please install to proceed." + fi +} +check_requirements + +### + +# DB functions + +# Get the last processed state from the database +get_state() { + sqlite3 "$DB" "$db_get_state" +} + +# Update the state database +set_state() { + local filename=$1 + local filetime=$2 + local filesize=$3 + local position=${4:-0} + local inode=$5 + db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode") + sqlite3 "$DB" "$db_set_state" +} + +### + +# Main parsing funstion +jvb_log_parse() { + + local file=$1 + local start_pos=$2 + new_last_pos="$start_pos" + + # Local assoc array for conference events tracking + declare -A start_times + + # Get size and position for progress tracking + local total_size=$(stat -c '%s' "$file") + local processed_lines=0 + local procesed_bytes=0 + + # We open the file and start reading from $start_pos bytes + exec 3<"$file" + while IFS= read -r line; do + # save new position (previous plus bytes in current line plus 1 for the new line) + new_last_pos=$((new_last_pos + ${#line} + 1)) + + # increment progress stats + processed_lines=$((processed_lines + 1)) + processed_bytes=$((processed_bytes + ${#line} + 1)) + + # show progress if in verbose mode + if [[ "$verbose" == true ]]; then + local percent=$((100 * processed_bytes / total_size)) + echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r" + fi + + # locate conference starting event +# if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then + if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then + timestamp="${BASH_REMATCH[1]}" + conferenceName="${BASH_REMATCH[2]}" + start_times["$conferenceName"]="$timestamp" + + # locate the corresponding conference ending event +# elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then + elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then + end_time="${BASH_REMATCH[1]}" + conferenceName="${BASH_REMATCH[2]}" + conferenceId="${BASH_REMATCH[3]}" + start_time="${start_times["$conferenceName"]}" + + if [[ -n "$start_time" ]]; then + db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time") + sqlite3 "$DB" "$db_insert" + unset start_times["$conferenceName"] + fi + fi + # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value + done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null) + + # Close the file descriptor + exec 3<&- +} + +### + +# commandline options + +while getopts ":dfcpv" opt; do + case $opt in + d) + cmd="--create-db" + ;; + f) + cmd="--flush" + ;; + c) + cmd="--check" + ;; + p) + cmd="--parse" + ;; + v) + verbose=true + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + echo -e "$help" + exit 1 + ;; + esac +done +shift $((OPTIND -1)) + +case "$cmd" in + + --create-db) + sqlite3 "$DB" "$db_create" + echo "Database created." + exit 0 + ;; + + --flush) + sqlite3 "$DB" "$db_flush" + echo "Tables flushed." + exit 0 + ;; + + --check) + + # Retrieve last log file and position inside it + IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state) + + # Initialize logfile vars + current_inode=$(stat -c '%i' "$LOGFILE") + current_filetime=$(stat -c '%Y' "$LOGFILE") + current_size=$(stat -c '%s' "$LOGFILE") + + if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then + echo "It looks like a fresh install. You can now run log parsing." + exit 0 + fi + + # report + echo "Last file: $last_file" + echo "Last filetime: $last_filetime" + echo "Last inode: $last_inode" + echo "Last size: $last_size" + echo "Last processed position: $last_pos" + echo "Current filetime: $current_filetime" + echo "Current inode: $current_inode" + echo "Current size: $current_size" + + if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then + echo "Log file has rotated." + else + echo "Log file has not rotated." + fi + + if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then + echo "New lines have been added to the log." + else + echo "No new lines in the log." + fi + exit 0 + ;; + + --parse) + + # Retrieve last log file and position inside it + IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state) + + # Initialize logfile vars + last_pos=${last_pos:-0} + current_inode=$(stat -c '%i' "$LOGFILE") + current_filetime=$(stat -c '%Y' "$LOGFILE") + current_size=$(stat -c '%s' "$LOGFILE") + + # Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate) + # parse the rotated log file + if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then + echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE" + jvb_log_parse "$ROTATED_LOGFILE" 0 "$verbose" + last_file="$ROTATED_LOGFILE" + last_inode=$(stat -c '%i' "$ROTATED_LOGFILE") + last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE") + set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode" + fi + + # parse the current log file + echo "Processing the current log file: $LOGFILE" + jvb_log_parse "$LOGFILE" "$last_pos" "$verbose" + + if [[ "$verbose" == true ]]; then + echo -e "\nNew last position after parsing: $new_last_pos" + fi + + # update the state in db + set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode" + + echo "Data import finished." + exit 0 + ;; + + *) + echo -e "$help" + exit 1 + ;; +esac