jilo/jvb/jitsi-stats

#!/usr/bin/env bash

### Variables

# log files
LOGFILE="./jvb.log"
ROTATED_LOGFILE="./jvb.log.1"
# SQLite database file
DB="./jitsi-stats.db"

###

# SQLite queries
db_get_state='SELECT filename, filetime, position, inode FROM state WHERE id = 1;'
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', position='%s', inode='%s' WHERE id = 1;"
db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');"
db_create="
DROP TABLE conferences;
CREATE TABLE conferences (
    id INTEGER PRIMARY_KEY,
    conference_name TEXT,
    conference_id TEXT,
    start TEXT,
    end TEXT
);
DROP TABLE state;
CREATE TABLE state (
    id INTEGER PRIMARY_KEY,
    time TEXT,
    filename TEXT,
    filetime INTEGER,
    position INTEGER CHECK(typeof(position)='integer'),
    inode INTEGER
);
INSERT OR REPLACE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);"
db_flush="
DELETE FROM conferences;
DELETE FROM state;
INSERT OR REPLACE INTO state (id, time, filename, filetime, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0);"

###

# DB functions

# Get the last processed state from the database
get_state() {
    sqlite3 "$DB" "$db_get_state"
}

# Update the state database
set_state() {
    local filename=$1
    local filetime=$2
    local position=${3:-0}
    local inode=$4
    db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$position" "$inode")
    sqlite3 "$DB" "$db_set_state"
}

###

# Main parsing funstion
jvb_log_parse() {

    local file=$1
    local start_pos=$2
    local new_last_pos="$start_pos"

    # Local assoc array for conference events tracking
    declare -A start_times

    # We open the file and start reading from $start_pos bytes
    exec 3<"$file"
    while IFS= read -r line; do
        # save new position (previous plus bytes in current line plus 1 for the new line)
        new_last_pos=$((new_last_pos + ${#line} + 1))

        # locate conference starting event
        if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
            timestamp="${BASH_REMATCH[1]}"
            conferenceId="${BASH_REMATCH[2]}"
            start_times["$conferenceId"]="$timestamp"

        # locate the corresponding conference ending event
        elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then
            end_time="${BASH_REMATCH[1]}"
            conferenceId="${BASH_REMATCH[2]}"
            conferenceName="${BASH_REMATCH[3]}"
            start_time="${start_times["$conferenceId"]}"

            if [[ -n "$start_time" ]]; then
                db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
                sqlite3 "$DB" "$db_insert"
                unset start_times["$conferenceId"]
            fi
        fi
    # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
    done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)

    # Close the file descriptor and return the new position value
    exec 3<&-
    echo "$new_last_pos"
}

###

# commandline options
case "$1" in

    --create-db|-d)
        sqlite3 "$DB" "$db_create"
        echo "Database created."
        exit 0
        ;;

    --flush|-f)
        sqlite3 "$DB" "$db_flush"
        echo "Tables flushed."
        exit 0
        ;;

    --check|-c)

        # Retrieve last log file and position inside it
        IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state)

        # Initialize logfile vars
        current_inode=$(stat -c '%i' "$LOGFILE")
        current_filetime=$(stat -c '%Y' "$LOGFILE")

        if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then
            echo "It looks like a fresh install. You can now run log parsing."
            exit 0
        fi

        # report
        echo "Last file: $last_file"
        echo "Last filetime: $last_filetime"
        echo "Last inode: $last_inode"
        echo "Last processed position: $last_pos"
        echo "Current filetime: $current_filetime"
        echo "Current inode: $current_inode"

        if [[ "$last_file" != "$LOGFILE" || "$last_inode" != "$current_inode" ]]; then
            echo "Log file has rotated."
        else
            echo "Log file has not rotated."
        fi

        if [[ "$current_filetime" -ne "$last_filetime" ]]; then
            echo "New lines have been added to the log."
        else
            echo "No new lines in the log."
        fi
        exit 0
        ;;

    --parse|-p)

        # Retrieve last log file and position inside it
        IFS='|' read -r last_file last_filetime last_pos last_inode <<< $(get_state)

        # Initialize logfile vars
        last_pos=${last_pos:-0}
        current_inode=$(stat -c '%i' "$LOGFILE")
        current_filetime=$(stat -c '%Y' "$LOGFILE")

        # Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate)
        # parse the rotated log file
        current_size=$(stat -c '%s' "$LOGFILE")
        if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
            echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE"
            last_pos=$(jvb_log_parse "$ROTATED_LOGFILE" 0)
            last_file="$ROTATED_LOGFILE"
            last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
            last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
            set_state "$last_file" "$last_filetime" "$last_pos" "$last_inode"
        fi

        # parse the current log file
        echo "Processing the current log file: $LOGFILE"
        new_last_pos=$(jvb_log_parse "$LOGFILE" "$last_pos")

        # DEBUG
        echo "New last position after parsing: $new_last_pos"

        # update the state in db
        set_state "$LOGFILE" "$current_filetime" "$new_last_pos" "$current_inode"

        echo "Data import finished."
        exit 0
        ;;

    *)
        echo -e "Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p - parse the logs"
        exit 1
        ;;
esac