Fixes errors, better command-line options and progress output.

jvb/jitsi-stats

@@ -11,11 +11,11 @@ DB="./jitsi-stats.db"
 ###
 
 # SQLite queries
-db_get_state='SELECT filename, filetime, position, inode FROM state WHERE id = 1;'
+db_get_state='SELECT filename, filetime, filesize, position, inode FROM state WHERE id = 1;'
-db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', position='%s', inode='%s' WHERE id = 1;"
+db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE id = 1;"
 db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');"
 db_create="
-DROP TABLE conferences;
+DROP TABLE IF EXISTS conferences;
 CREATE TABLE conferences (
     id INTEGER PRIMARY_KEY,
     conference_name TEXT,
@@ -23,20 +23,23 @@ CREATE TABLE conferences (
     start TEXT,
     end TEXT
 );
-DROP TABLE state;
+DROP TABLE IF EXISTS state;
 CREATE TABLE state (
     id INTEGER PRIMARY_KEY,
     time TEXT,
     filename TEXT,
     filetime INTEGER,
+    filesize INTEGER,
     position INTEGER CHECK(typeof(position)='integer'),
     inode INTEGER
 );
-INSERT OR REPLACE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);"
+INSERT OR REPLACE INTO state (id, time, filename, filetime, filesize, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
 db_flush="
 DELETE FROM conferences;
 DELETE FROM state;
-INSERT OR REPLACE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);"
+INSERT OR REPLACE INTO state (id, time, filename, filetime, filesize, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
+
+help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]"
 
 ###
 
@@ -51,9 +54,10 @@ get_state() {
 set_state() {
     local filename=$1
     local filetime=$2
-    local position=${3:-0}
+    local filesize=$3
-    local inode=$4
+    local position=${4:-0}
-    db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$position" "$inode")
+    local inode=$5
+    db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode")
     sqlite3 "$DB" "$db_set_state"
 }
 
@@ -64,17 +68,32 @@ jvb_log_parse() {
 
     local file=$1
     local start_pos=$2
-    local new_last_pos="$start_pos"
+    new_last_pos="$start_pos"
 
     # Local assoc array for conference events tracking
     declare -A start_times
 
+    # Get size and position for progress tracking
+    local total_size=$(stat -c '%s' "$file")
+    local processed_lines=0
+    local procesed_bytes=0
+
     # We open the file and start reading from $start_pos bytes
     exec 3<"$file"
     while IFS= read -r line; do
         # save new position (previous plus bytes in current line plus 1 for the new line)
         new_last_pos=$((new_last_pos + ${#line} + 1))
 
+        # increment progress stats
+        processed_lines=$((processed_lines + 1))
+        processed_bytes=$((processed_bytes + ${#line} + 1))
+
+        # show progress if in verbose mode
+        if [[ "$verbose" == true ]]; then
+            local percent=$((100 * processed_bytes / total_size))
+            echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
+        fi
+
         # locate conference starting event
         if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
             timestamp="${BASH_REMATCH[1]}"
@@ -97,36 +116,63 @@ jvb_log_parse() {
     # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
     done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
 
-    # Close the file descriptor and return the new position value
+    # Close the file descriptor
     exec 3<&-
-    echo "$new_last_pos"
 }
 
 ###
 
 # commandline options
-case "$1" in
 
-    --create-db|-d)
+while getopts ":dfcpv" opt; do
+    case $opt in
+        d)
+            cmd="--create-db"
+            ;;
+        f)
+            cmd="--flush"
+            ;;
+        c)
+            cmd="--check"
+            ;;
+        p)
+            cmd="--parse"
+            ;;
+        v)
+            verbose=true
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG" >&2
+            echo -e "$help"
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+case "$cmd" in
+
+    --create-db)
         sqlite3 "$DB" "$db_create"
         echo "Database created."
         exit 0
         ;;
 
-    --flush|-f)
+    --flush)
         sqlite3 "$DB" "$db_flush"
         echo "Tables flushed."
         exit 0
         ;;
 
-    --check|-c)
+    --check)
 
         # Retrieve last log file and position inside it
-        IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state)
+        IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)
 
         # Initialize logfile vars
         current_inode=$(stat -c '%i' "$LOGFILE")
         current_filetime=$(stat -c '%Y' "$LOGFILE")
+        current_size=$(stat -c '%s' "$LOGFILE")
 
         if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then
             echo "It looks like a fresh install. You can now run log parsing."
@@ -137,17 +183,19 @@ case "$1" in
         echo "Last file: $last_file"
         echo "Last filetime: $last_filetime"
         echo "Last inode: $last_inode"
+        echo "Last size: $last_size"
         echo "Last processed position: $last_pos"
         echo "Current filetime: $current_filetime"
         echo "Current inode: $current_inode"
+        echo "Current size: $current_size"
 
-        if [[ "$last_file" != "$LOGFILE" || "$last_inode" != "$current_inode" ]]; then
+        if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
             echo "Log file has rotated."
         else
             echo "Log file has not rotated."
         fi
 
-        if [[ "$current_filetime" -ne "$last_filetime" ]]; then
+        if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then
             echo "New lines have been added to the log."
         else
             echo "No new lines in the log."
@@ -155,44 +203,45 @@ case "$1" in
         exit 0
         ;;
 
-    --parse|-p)
+    --parse)
 
         # Retrieve last log file and position inside it
-        IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state)
+        IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)
 
         # Initialize logfile vars
         last_pos=${last_pos:-0}
         current_inode=$(stat -c '%i' "$LOGFILE")
         current_filetime=$(stat -c '%Y' "$LOGFILE")
+        current_size=$(stat -c '%s' "$LOGFILE")
 
         # Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate)
         # parse the rotated log file
-        current_size=$(stat -c '%s' "$LOGFILE")
         if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
             echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE"
-            last_pos=$(jvb_log_parse "$ROTATED_LOGFILE" 0)
+            jvb_log_parse "$ROTATED_LOGFILE" 0 "$verbose"
             last_file="$ROTATED_LOGFILE"
             last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
             last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
-            set_state "$last_file" "$last_filetime" "$last_pos" "$last_inode"
+            set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode"
         fi
 
         # parse the current log file
         echo "Processing the current log file: $LOGFILE"
-        new_last_pos=$(jvb_log_parse "$LOGFILE" "$last_pos")
+        jvb_log_parse "$LOGFILE" "$last_pos" "$verbose"
 
-        # DEBUG
+        if [[ "$verbose" == true ]]; then
-        echo "New last position after parsing: $new_last_pos"
+            echo -e "\nNew last position after parsing: $new_last_pos"
+        fi
 
         # update the state in db
-        set_state "$LOGFILE" "$current_filetime" "$new_last_pos" "$current_inode"
+        set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode"
 
         echo "Data import finished."
         exit 0
         ;;
 
     *)
-        echo -e "Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p - parse the logs"
+        echo -e "$help"
         exit 1
         ;;
 esac