diff --git a/jicofo/jicofo-stats b/jicofo/jicofo-stats deleted file mode 100755 index 10a26ed..0000000 --- a/jicofo/jicofo-stats +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env bash - -### Variables - -# log files -LOGFILE="./jicofo.log" -ROTATED_LOGFILE="./jicofo.log.1" -# SQLite database file -DB="./jitsi-stats.db" - -### - -# SQLite queries -db_get_state="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = 'JICOFO';" -db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = 'JICOFO';" -db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');" -db_create=" -DROP TABLE IF EXISTS conferences; -CREATE TABLE conferences ( - id INTEGER PRIMARY_KEY, - conference_name TEXT, - conference_id TEXT, - start TEXT, - end TEXT -); -DROP TABLE IF EXISTS state; -CREATE TABLE state ( - id INTEGER PRIMARY_KEY, - jitsi_component TEXT, - time TEXT, - filename TEXT, - filetime INTEGER, - filesize INTEGER, - position INTEGER CHECK(typeof(position)='integer'), - inode INTEGER -); -INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);" -db_flush=" -DELETE FROM conferences; -DELETE FROM state; -INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);" - -help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]" - -### - -# First we check for requirements -check_requirements() { - # required programs, anything non-bash - edit as needed - # deb packages - sqlite3, coreutils(stat,dd) - local required_programs=("sqlite3" "stat" "dd") - local requirements_missing='' - for program in "${required_programs[@]}"; do - if ! command -v "$program" &> /dev/null; then - requirements_missing+="$program, " - fi - done - if [[ "$requirements_missing" != '' ]]; then - requirements_missing=${requirements_missing::-2} - echo "Error: $requirements_missing - not found. Please install to proceed." - fi -} -check_requirements - -### - -# DB functions - -# Get the last processed state from the database -get_state() { - sqlite3 "$DB" "$db_get_state" -} - -# Update the state database -set_state() { - local filename=$1 - local filetime=$2 - local filesize=$3 - local position=${4:-0} - local inode=$5 - db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode") - sqlite3 "$DB" "$db_set_state" -} - -### - -# Main parsing funstion -jicofo_log_parse() { - - local file=$1 - local start_pos=$2 - new_last_pos="$start_pos" - - # Local assoc array for conference events tracking - declare -A start_times - - # Get size and position for progress tracking - local total_size=$(stat -c '%s' "$file") - local processed_lines=0 - local procesed_bytes=0 - - # We open the file and start reading from $start_pos bytes - exec 3<"$file" - while IFS= read -r line; do - # save new position (previous plus bytes in current line plus 1 for the new line) - new_last_pos=$((new_last_pos + ${#line} + 1)) - - # increment progress stats - processed_lines=$((processed_lines + 1)) - processed_bytes=$((processed_bytes + ${#line} + 1)) - - # show progress if in verbose mode - if [[ "$verbose" == true ]]; then - local percent=$((100 * processed_bytes / total_size)) - echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r" - fi - - # locate conference starting event - if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then - timestamp="${BASH_REMATCH[1]}" - conferenceName="${BASH_REMATCH[2]}" - start_times["$conferenceName"]="$timestamp" - - # locate the corresponding conference ending event - elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then - end_time="${BASH_REMATCH[1]}" - conferenceName="${BASH_REMATCH[2]}" - conferenceId="${BASH_REMATCH[3]}" - start_time="${start_times["$conferenceName"]}" - - if [[ -n "$start_time" ]]; then - db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time") - sqlite3 "$DB" "$db_insert" - unset start_times["$conferenceName"] - fi - fi - # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value - done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null) - - # Close the file descriptor - exec 3<&- -} - -### - -# commandline options - -while getopts ":dfcpv" opt; do - case $opt in - d) - cmd="--create-db" - ;; - f) - cmd="--flush" - ;; - c) - cmd="--check" - ;; - p) - cmd="--parse" - ;; - v) - verbose=true - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - echo -e "$help" - exit 1 - ;; - esac -done -shift $((OPTIND -1)) - -case "$cmd" in - - --create-db) - sqlite3 "$DB" "$db_create" - echo "Database created." - exit 0 - ;; - - --flush) - sqlite3 "$DB" "$db_flush" - echo "Tables flushed." - exit 0 - ;; - - --check) - - # Retrieve last log file and position inside it - IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state) - - # Initialize logfile vars - current_inode=$(stat -c '%i' "$LOGFILE") - current_filetime=$(stat -c '%Y' "$LOGFILE") - current_size=$(stat -c '%s' "$LOGFILE") - - if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then - echo "It looks like a fresh install. You can now run log parsing." - exit 0 - fi - - # report - echo "Last file: $last_file" - echo "Last filetime: $last_filetime" - echo "Last inode: $last_inode" - echo "Last size: $last_size" - echo "Last processed position: $last_pos" - echo "Current filetime: $current_filetime" - echo "Current inode: $current_inode" - echo "Current size: $current_size" - - if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then - echo "Log file has rotated." - else - echo "Log file has not rotated." - fi - - if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then - echo "New lines have been added to the log." - else - echo "No new lines in the log." - fi - exit 0 - ;; - - --parse) - - # Retrieve last log file and position inside it - IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state) - - # Initialize logfile vars - last_pos=${last_pos:-0} - current_inode=$(stat -c '%i' "$LOGFILE") - current_filetime=$(stat -c '%Y' "$LOGFILE") - current_size=$(stat -c '%s' "$LOGFILE") - - # Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate) - # parse the rotated log file - if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then - echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE" - jvb_log_parse "$ROTATED_LOGFILE" 0 "$verbose" - last_file="$ROTATED_LOGFILE" - last_inode=$(stat -c '%i' "$ROTATED_LOGFILE") - last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE") - set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode" - fi - - # parse the current log file - echo "Processing the current log file: $LOGFILE" - jvb_log_parse "$LOGFILE" "$last_pos" "$verbose" - - if [[ "$verbose" == true ]]; then - echo -e "\nNew last position after parsing: $new_last_pos" - fi - - # update the state in db - set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode" - - echo "Data import finished." - exit 0 - ;; - - *) - echo -e "$help" - exit 1 - ;; -esac diff --git a/jvb/jilo.conf b/jilo.conf similarity index 100% rename from jvb/jilo.conf rename to jilo.conf diff --git a/jvb/jitsi-stats b/jitsi-stats similarity index 71% rename from jvb/jitsi-stats rename to jitsi-stats index 2e81b36..68ed556 100755 --- a/jvb/jitsi-stats +++ b/jitsi-stats @@ -30,12 +30,13 @@ DB=${DB:-$DEFAULT_DB} # SQLite queries db_get_state_template="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = '%s';" db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = '%s';" -db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');" +db_insert_template="INSERT INTO conferences (jitsi_component, conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s', '%s');" db_drop=" DROP TABLE IF EXISTS conferences; DROP TABLE IF EXISTS state;" db_create="CREATE TABLE conferences ( id INTEGER PRIMARY_KEY, + jitsi_component TEXT, conference_name TEXT, conference_id TEXT, start TEXT, @@ -104,7 +105,7 @@ set_state() { ### -# Main parsing funstion +# Jitsi Videobridge parsing funstion jvb_log_parse() { local file=$1 @@ -119,6 +120,94 @@ jvb_log_parse() { local processed_lines=0 local procesed_bytes=0 + # We open the file and start reading from $start_pos bytes + exec 3<"$file" + while IFS= read -r line; do + # save new position (previous plus bytes in current line plus 1 for the new line) + new_last_pos=$((new_last_pos + ${#line} + 1)) + + # increment progress stats + processed_lines=$((processed_lines + 1)) + processed_bytes=$((processed_bytes + ${#line} + 1)) + + # show progress if in verbose mode + if [[ "$verbose" == true ]]; then + local percent=$((100 * processed_bytes / total_size)) + echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r" + fi + + case $jitsi_component in + + JVB) + # locate conference starting event + if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then + timestamp="${BASH_REMATCH[1]}" + conferenceId="${BASH_REMATCH[2]}" + start_times["$conferenceId"]="$timestamp" + + # locate the corresponding conference ending event + elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then + end_time="${BASH_REMATCH[1]}" + conferenceId="${BASH_REMATCH[2]}" + conferenceName="${BASH_REMATCH[3]}" + start_time="${start_times["$conferenceId"]}" + + if [[ -n "$start_time" ]]; then + db_insert=$(printf "$db_insert_template" "$jitsi_component" "$conferenceName" "$conferenceId" "$start_time" "$end_time") + sqlite3 "$DB" "$db_insert" + unset start_times["$conferenceId"] + fi + fi + ;; + + JICOFO) + # locate conference starting event + if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then + timestamp="${BASH_REMATCH[1]}" + conferenceName="${BASH_REMATCH[2]}" + start_times["$conferenceName"]="$timestamp" + + # locate the corresponding conference ending event + elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then + end_time="${BASH_REMATCH[1]}" + conferenceName="${BASH_REMATCH[2]}" + conferenceId="${BASH_REMATCH[3]}" + start_time="${start_times["$conferenceName"]}" + + if [[ -n "$start_time" ]]; then + db_insert=$(printf "$db_insert_template" "$jitsi_component" "$conferenceName" "$conferenceId" "$start_time" "$end_time") + sqlite3 "$DB" "$db_insert" + unset start_times["$conferenceName"] + fi + fi + ;; + + esac + + # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value + done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null) + + # Close the file descriptor + exec 3<&- +} + +### + +# Jicofo parsing funstion +jicofo_log_parse() { + + local file=$1 + local start_pos=$2 + new_last_pos="$start_pos" + + # Local assoc array for conference events tracking + declare -A start_times + + # Get size and position for progress tracking + local total_size=$(stat -c '%s' "$file") + local processed_lines=0 + local procesed_bytes=0 + # We open the file and start reading from $start_pos bytes exec 3<"$file" while IFS= read -r line; do @@ -136,22 +225,22 @@ jvb_log_parse() { fi # locate conference starting event - if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then + if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then timestamp="${BASH_REMATCH[1]}" - conferenceId="${BASH_REMATCH[2]}" - start_times["$conferenceId"]="$timestamp" + conferenceName="${BASH_REMATCH[2]}" + start_times["$conferenceName"]="$timestamp" # locate the corresponding conference ending event - elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then + elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then end_time="${BASH_REMATCH[1]}" - conferenceId="${BASH_REMATCH[2]}" - conferenceName="${BASH_REMATCH[3]}" - start_time="${start_times["$conferenceId"]}" + conferenceName="${BASH_REMATCH[2]}" + conferenceId="${BASH_REMATCH[3]}" + start_time="${start_times["$conferenceName"]}" if [[ -n "$start_time" ]]; then db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time") sqlite3 "$DB" "$db_insert" - unset start_times["$conferenceId"] + unset start_times["$conferenceName"] fi fi # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value @@ -161,7 +250,6 @@ jvb_log_parse() { exec 3<&- } -### ### FIXME - this is not currently used # check if and which process is running @@ -253,7 +341,7 @@ case "$cmd" in for jitsi_component in "${jitsi_components[@]}"; do # Retrieve last log file and position inside it - IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state) + IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state "$jitsi_component") # Initialize logfile vars LOGFILE=$(eval "echo \$${jitsi_component}_LOGFILE") @@ -284,9 +372,9 @@ case "$cmd" in fi if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then - echo "New lines have been added to the log." + echo -e "New lines have been added to the log.\n" else - echo "No new lines in the log." + echo -e "No new lines in the log.\n" fi done fi