New version, uses DB instead of a statefile.

jvb/jitsi-stats

@@ -2,59 +2,104 @@
 
 #grep -e 'Videobridge.createConference\|Conference.expire' jvb.log > 11
 
-LOGFILE="/var/log/jitsi/jvb.log"
+### Variables
-STATEFILE="/srv/jitsi/bin/jitsi-stats.state"
-DB="/srv/jitsi/data/jitsi-stats.db"
 
-if [[ -f "$STATEFILE" ]]; then
+# log files
-    last_processed=$(cat "$STATEFILE")
+LOGFILE="../../jvb.log"
-else
+ROTATED_LOGFILE="../../jvb.log.1"
-    last_processed="1970-01-01 00:00:00.000"
+# SQLite database file
-fi
+DB="../../jitsi-stats.db"
 
+###
+
+# SQLite queries
+db_get_state='SELECT filename, filetime, position, inode FROM state WHERE id = 1;'
+db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', position='%s', inode='%s' WHERE id = 1;"
+db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');"
+
+# Get the last processed state from the database
+get_state() {
+    sqlite3 "$DB" "$db_get_state"
+}
+
+# Update the state database
+set_state() {
+    local filename=$1
+    local filetime=$2
+    local position=${3:-0}
+    local inode=$4
+    db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$position" "$inode")
+    sqlite3 "$DB" "$db_set_state"
+}
+
+# Retrieve last log file and position inside it
+IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state)
+
+# Initialize logfile vars
+last_pos=${last_pos:-0}
+current_inode=$(stat -c '%i' "$LOGFILE")
+current_filetime=$(stat -c '%Y' "$LOGFILE")
+
+# Local assoc array for conference events tracking
 declare -A start_times
-declare -A conference_names
 
 jvb_log_parse() {
 
-    local new_last_processed="$last_processed"
+    local file=$1
+    local start_pos=$2
+    local new_last_pos="$start_pos"
 
+    # We open the file and start reading from $start_pos bytes
+    exec 3<"$file"
     while IFS= read -r line; do
-        if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9-]+) ]]; then
+        # save new position (previous plus bytes in current line plus 1 for the new line)
+        new_last_pos=$((new_last_pos + ${#line} + 1))
+
+        # locate conference starting event
+        if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
             timestamp="${BASH_REMATCH[1]}"
             conferenceId="${BASH_REMATCH[2]}"
-            if [[ "$timestamp" > "$last_processed" ]]; then
             start_times["$conferenceId"]="$timestamp"
-                if [[ "$timestamp" > "$new_last_processed" ]]; then
-                    new_last_processed="$timestamp"
-                fi
-            fi
 
-        elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9-]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then
+        # locate the corresponding conference ending event
-            timestamp="${BASH_REMATCH[1]}"
+        elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then
+            end_time="${BASH_REMATCH[1]}"
             conferenceId="${BASH_REMATCH[2]}"
             conferenceName="${BASH_REMATCH[3]}"
-            if [[ "$timestamp" > "$last_processed" ]]; then
             start_time="${start_times["$conferenceId"]}"
-                conference_names["$conferenceId"]="$conferenceName"
+
             if [[ -n "$start_time" ]]; then
-                        echo "$conferenceName $eventId $start_time $timestamp"
+                db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
-                        sqlite3 "$DB" <<EOF
+                sqlite3 "$DB" "$db_insert"
-                            INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('$conferenceName', '$conferenceId', '$start_time', '$timestamp');
-EOF
                 unset start_times["$conferenceId"]
-                        unset conference_names["$conferenceId"]
-                    fi
-                    if [[ "$fimestamp" > "$new_last_processed" ]]; then
-                        new_last_processed="$timestamp"
-                    fi
             fi
         fi
+    # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
+    done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)
 
-    done < "$LOGFILE"
+    # Close the file descriptor and return the new position value
-
+    exec 3<&-
-    echo "$new_last_processed" > "$STATEFILE"
+    echo "$new_last_pos"
-
 }
 
-jvb_log_parse
+# parse the rotated log file
+if [[ "$last_file" == "$LOGFILE" && -f "$ROTATED_LOGFILE" ]]; then
+    echo "Processing the rotated log file: $ROTATED_LOGFILE"
+    last_pos=$(jvb_log_parse "$ROTATED_LOGFILE" 0)
+    last_file="$ROTATED_LOGFILE"
+    last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
+    last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
+    set_state "$last_file" "$last_filetime" "$last_pos" "$last_inode"
+fi
+
+# parse the current log file
+echo "Processing the current log file: $LOGFILE"
+new_last_pos=$(jvb_log_parse "$LOGFILE" "$last_pos")
+
+# DEBUG
+echo "New last position after parsing: $new_last_pos"
+
+# update the state in db
+set_state "$LOGFILE" "$current_filetime" "$new_last_pos" "$current_inode"
+
+echo "Data import finished."

jvb/jitsi-stats.schema

@@ -5,3 +5,12 @@ CREATE TABLE IF NOT EXISTS conferences (
     start TEXT,
     end TEXT
 );
+CREATE TABLE IF NOT EXISTS state (
+    id INTEGER PRIMARY_KEY,
+    time TEXT,
+    filename TEXT,
+    filetime INTEGER,
+    position INTEGER CHECK(typeof(position)='integer'),
+    inode INTEGER
+);
+INSERT OR IGNORE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);