diff --git a/jvb/jitsi-stats b/jvb/jitsi-stats index 4f050ef..320c39d 100755 --- a/jvb/jitsi-stats +++ b/jvb/jitsi-stats @@ -11,11 +11,11 @@ DB="./jitsi-stats.db" ### # SQLite queries -db_get_state='SELECT filename, filetime, position, inode FROM state WHERE id = 1;' -db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', position='%s', inode='%s' WHERE id = 1;" +db_get_state='SELECT filename, filetime, filesize, position, inode FROM state WHERE id = 1;' +db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE id = 1;" db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');" db_create=" -DROP TABLE conferences; +DROP TABLE IF EXISTS conferences; CREATE TABLE conferences ( id INTEGER PRIMARY_KEY, conference_name TEXT, @@ -23,20 +23,23 @@ CREATE TABLE conferences ( start TEXT, end TEXT ); -DROP TABLE state; +DROP TABLE IF EXISTS state; CREATE TABLE state ( id INTEGER PRIMARY_KEY, time TEXT, filename TEXT, filetime INTEGER, + filesize INTEGER, position INTEGER CHECK(typeof(position)='integer'), inode INTEGER ); -INSERT OR REPLACE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);" +INSERT OR REPLACE INTO state (id, time, filename, filetime, filesize, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);" db_flush=" DELETE FROM conferences; DELETE FROM state; -INSERT OR REPLACE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);" +INSERT OR REPLACE INTO state (id, time, filename, filetime, filesize, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);" + +help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]" ### @@ -51,9 +54,10 @@ get_state() { set_state() { local filename=$1 local filetime=$2 - local position=${3:-0} - local inode=$4 - db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$position" "$inode") + local filesize=$3 + local position=${4:-0} + local inode=$5 + db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode") sqlite3 "$DB" "$db_set_state" } @@ -64,17 +68,32 @@ jvb_log_parse() { local file=$1 local start_pos=$2 - local new_last_pos="$start_pos" + new_last_pos="$start_pos" # Local assoc array for conference events tracking declare -A start_times + # Get size and position for progress tracking + local total_size=$(stat -c '%s' "$file") + local processed_lines=0 + local procesed_bytes=0 + # We open the file and start reading from $start_pos bytes exec 3<"$file" while IFS= read -r line; do # save new position (previous plus bytes in current line plus 1 for the new line) new_last_pos=$((new_last_pos + ${#line} + 1)) + # increment progress stats + processed_lines=$((processed_lines + 1)) + processed_bytes=$((processed_bytes + ${#line} + 1)) + + # show progress if in verbose mode + if [[ "$verbose" == true ]]; then + local percent=$((100 * processed_bytes / total_size)) + echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r" + fi + # locate conference starting event if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then timestamp="${BASH_REMATCH[1]}" @@ -97,36 +116,63 @@ jvb_log_parse() { # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null) - # Close the file descriptor and return the new position value + # Close the file descriptor exec 3<&- - echo "$new_last_pos" } ### # commandline options -case "$1" in - --create-db|-d) +while getopts ":dfcpv" opt; do + case $opt in + d) + cmd="--create-db" + ;; + f) + cmd="--flush" + ;; + c) + cmd="--check" + ;; + p) + cmd="--parse" + ;; + v) + verbose=true + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + echo -e "$help" + exit 1 + ;; + esac +done +shift $((OPTIND -1)) + +case "$cmd" in + + --create-db) sqlite3 "$DB" "$db_create" echo "Database created." exit 0 ;; - --flush|-f) + --flush) sqlite3 "$DB" "$db_flush" echo "Tables flushed." exit 0 ;; - --check|-c) + --check) # Retrieve last log file and position inside it - IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state) + IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state) # Initialize logfile vars current_inode=$(stat -c '%i' "$LOGFILE") current_filetime=$(stat -c '%Y' "$LOGFILE") + current_size=$(stat -c '%s' "$LOGFILE") if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then echo "It looks like a fresh install. You can now run log parsing." @@ -137,17 +183,19 @@ case "$1" in echo "Last file: $last_file" echo "Last filetime: $last_filetime" echo "Last inode: $last_inode" + echo "Last size: $last_size" echo "Last processed position: $last_pos" echo "Current filetime: $current_filetime" echo "Current inode: $current_inode" + echo "Current size: $current_size" - if [[ "$last_file" != "$LOGFILE" || "$last_inode" != "$current_inode" ]]; then + if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then echo "Log file has rotated." else echo "Log file has not rotated." fi - if [[ "$current_filetime" -ne "$last_filetime" ]]; then + if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then echo "New lines have been added to the log." else echo "No new lines in the log." @@ -155,44 +203,45 @@ case "$1" in exit 0 ;; - --parse|-p) + --parse) # Retrieve last log file and position inside it - IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state) + IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state) # Initialize logfile vars last_pos=${last_pos:-0} current_inode=$(stat -c '%i' "$LOGFILE") current_filetime=$(stat -c '%Y' "$LOGFILE") + current_size=$(stat -c '%s' "$LOGFILE") # Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate) # parse the rotated log file - current_size=$(stat -c '%s' "$LOGFILE") if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE" - last_pos=$(jvb_log_parse "$ROTATED_LOGFILE" 0) + jvb_log_parse "$ROTATED_LOGFILE" 0 "$verbose" last_file="$ROTATED_LOGFILE" last_inode=$(stat -c '%i' "$ROTATED_LOGFILE") last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE") - set_state "$last_file" "$last_filetime" "$last_pos" "$last_inode" + set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode" fi # parse the current log file echo "Processing the current log file: $LOGFILE" - new_last_pos=$(jvb_log_parse "$LOGFILE" "$last_pos") + jvb_log_parse "$LOGFILE" "$last_pos" "$verbose" - # DEBUG - echo "New last position after parsing: $new_last_pos" + if [[ "$verbose" == true ]]; then + echo -e "\nNew last position after parsing: $new_last_pos" + fi # update the state in db - set_state "$LOGFILE" "$current_filetime" "$new_last_pos" "$current_inode" + set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode" echo "Data import finished." exit 0 ;; *) - echo -e "Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p - parse the logs" + echo -e "$help" exit 1 ;; esac