jilo/jvb/jitsi-stats

#!/usr/bin/env bash

### Variables

# log files
LOGFILE="./jvb.log"
ROTATED_LOGFILE="./jvb.log.1"
# SQLite database file
DB="./jitsi-stats.db"

###

# SQLite queries
db_get_state='SELECT filename, filetime, filesize, position, inode FROM state WHERE id = 1;'
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE id = 1;"
db_insert_template="INSERT INTO conferences (conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s');"
db_create="
DROP TABLE IF EXISTS conferences;
CREATE TABLE conferences (
    id INTEGER PRIMARY_KEY,
    conference_name TEXT,
    conference_id TEXT,
    start TEXT,
    end TEXT
);
DROP TABLE IF EXISTS state;
CREATE TABLE state (
    id INTEGER PRIMARY_KEY,
    time TEXT,
    filename TEXT,
    filetime INTEGER,
    filesize INTEGER,
    position INTEGER CHECK(typeof(position)='integer'),
    inode INTEGER
);
INSERT OR REPLACE INTO state (id, time, filename, filetime, filesize, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
db_flush="
DELETE FROM conferences;
DELETE FROM state;
INSERT OR REPLACE INTO state (id, time, filename, filetime, filesize, position, inode) VALUES (1, '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"

help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]"

###

# First we check for requirements
check_requirements() {
    # required programs, anything non-bash - edit as needed
    # deb packages - sqlite3, coreutils(stat,dd)
    local required_programs=("sqlite3" "stat" "dd")
    local requirements_missing=''
    for program in "${required_programs[@]}"; do
        if ! command -v "$program" &> /dev/null; then
            requirements_missing+="$program, "
        fi
    done
    if [[ "$requirements_missing" != '' ]]; then
        requirements_missing=${requirements_missing::-2}
        echo "Error: $requirements_missing - not found. Please install to proceed."
    fi
}
check_requirements

###

# DB functions

# Get the last processed state from the database
get_state() {
    sqlite3 "$DB" "$db_get_state"
}

# Update the state database
set_state() {
    local filename=$1
    local filetime=$2
    local filesize=$3
    local position=${4:-0}
    local inode=$5
    db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode")
    sqlite3 "$DB" "$db_set_state"
}

###

# Main parsing funstion
jvb_log_parse() {

    local file=$1
    local start_pos=$2
    new_last_pos="$start_pos"

    # Local assoc array for conference events tracking
    declare -A start_times

    # Get size and position for progress tracking
    local total_size=$(stat -c '%s' "$file")
    local processed_lines=0
    local procesed_bytes=0

    # We open the file and start reading from $start_pos bytes
    exec 3<"$file"
    while IFS= read -r line; do
        # save new position (previous plus bytes in current line plus 1 for the new line)
        new_last_pos=$((new_last_pos + ${#line} + 1))

        # increment progress stats
        processed_lines=$((processed_lines + 1))
        processed_bytes=$((processed_bytes + ${#line} + 1))

        # show progress if in verbose mode
        if [[ "$verbose" == true ]]; then
            local percent=$((100 * processed_bytes / total_size))
            echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
        fi

        # locate conference starting event
        if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
            timestamp="${BASH_REMATCH[1]}"
            conferenceId="${BASH_REMATCH[2]}"
            start_times["$conferenceId"]="$timestamp"

        # locate the corresponding conference ending event
        elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then
            end_time="${BASH_REMATCH[1]}"
            conferenceId="${BASH_REMATCH[2]}"
            conferenceName="${BASH_REMATCH[3]}"
            start_time="${start_times["$conferenceId"]}"

            if [[ -n "$start_time" ]]; then
                db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
                sqlite3 "$DB" "$db_insert"
                unset start_times["$conferenceId"]
            fi
        fi
    # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
    done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)

    # Close the file descriptor
    exec 3<&-
}

###

# commandline options

while getopts ":dfcpv" opt; do
    case $opt in
        d)
            cmd="--create-db"
            ;;
        f)
            cmd="--flush"
            ;;
        c)
            cmd="--check"
            ;;
        p)
            cmd="--parse"
            ;;
        v)
            verbose=true
            ;;
        \?)
            echo "Invalid option: -$OPTARG" >&2
            echo -e "$help"
            exit 1
            ;;
    esac
done
shift $((OPTIND -1))

case "$cmd" in

    --create-db)
        sqlite3 "$DB" "$db_create"
        echo "Database created."
        exit 0
        ;;

    --flush)
        sqlite3 "$DB" "$db_flush"
        echo "Tables flushed."
        exit 0
        ;;

    --check)

        # Retrieve last log file and position inside it
        IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)

        # Initialize logfile vars
        current_inode=$(stat -c '%i' "$LOGFILE")
        current_filetime=$(stat -c '%Y' "$LOGFILE")
        current_size=$(stat -c '%s' "$LOGFILE")

        if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then
            echo "It looks like a fresh install. You can now run log parsing."
            exit 0
        fi

        # report
        echo "Last file: $last_file"
        echo "Last filetime: $last_filetime"
        echo "Last inode: $last_inode"
        echo "Last size: $last_size"
        echo "Last processed position: $last_pos"
        echo "Current filetime: $current_filetime"
        echo "Current inode: $current_inode"
        echo "Current size: $current_size"

        if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
            echo "Log file has rotated."
        else
            echo "Log file has not rotated."
        fi

        if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then
            echo "New lines have been added to the log."
        else
            echo "No new lines in the log."
        fi
        exit 0
        ;;

    --parse)

        # Retrieve last log file and position inside it
        IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state)

        # Initialize logfile vars
        last_pos=${last_pos:-0}
        current_inode=$(stat -c '%i' "$LOGFILE")
        current_filetime=$(stat -c '%Y' "$LOGFILE")
        current_size=$(stat -c '%s' "$LOGFILE")

        # Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate)
        # parse the rotated log file
        if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
            echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE"
            jvb_log_parse "$ROTATED_LOGFILE" 0 "$verbose"
            last_file="$ROTATED_LOGFILE"
            last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
            last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
            set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode"
        fi

        # parse the current log file
        echo "Processing the current log file: $LOGFILE"
        jvb_log_parse "$LOGFILE" "$last_pos" "$verbose"

        if [[ "$verbose" == true ]]; then
            echo -e "\nNew last position after parsing: $new_last_pos"
        fi

        # update the state in db
        set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode"

        echo "Data import finished."
        exit 0
        ;;

    *)
        echo -e "$help"
        exit 1
        ;;
esac