jilo/jilo

#!/usr/bin/env bash

###
# JItsi Log Observer
#
# Bash script for Jitsi Meet components (Videobridge, Jicofo, etc.) logs parsing
###


### Default configuration

# default log files and processes
DEFAULT_JVB_LOGFILE="/var/log/jitsi/jvb.log"
DEFAULT_JVB_PROCESS="videobridge"
DEFAULT_JICOFO_LOGFILE="/var/log/jitsi/jicofo.log"
DEFAULT_JICOFO_PROCESS="jicofo"
# Default SQLite database file
DEFAULT_DB="./jilo.db"

# Configuration file
CONFIG_FILE="./jilo.conf"

# Load configurations from the config file if it exists
if [[ -f "$CONFIG_FILE" ]]; then
    source "$CONFIG_FILE"
fi

# use default values if not overriden by config file
JVB_LOGFILE=${JVB_LOGFILE:-$DEFAULT_JVB_LOGFILE}
JVB_PROCESS=${JVB_PROCESS:-$DEFAULT_JVB_PROCESS}
JICOFO_LOGFILE=${JICOFO_LOGFILE:-$DEFAULT_JICOFO_LOGFILE}
JICOFO_PROCESS=${JICOFO_PROCESS:-$DEFAULT_JICOFO_PROCESS}
DB=${DB:-$DEFAULT_DB}

###

# SQLite queries
db_get_state_template="SELECT filename, filetime, filesize, position, inode FROM state WHERE jitsi_component = '%s';"
db_set_state_template="UPDATE state SET time=datetime('now'), filename='%s', filetime='%s', filesize='%s', position='%s', inode='%s' WHERE jitsi_component = '%s';"
db_insert_template="INSERT INTO conferences (jitsi_component, conference_name, conference_id, start, end) VALUES ('%s', '%s', '%s', '%s', '%s');"
db_drop="
DROP TABLE IF EXISTS conferences;
DROP TABLE IF EXISTS state;"
db_create="CREATE TABLE conferences (
    id INTEGER PRIMARY_KEY,
    jitsi_component TEXT,
    conference_name TEXT,
    conference_id TEXT,
    start TEXT,
    end TEXT
);
CREATE TABLE state (
    id INTEGER PRIMARY_KEY,
    jitsi_component TEXT,
    time TEXT,
    filename TEXT,
    filetime INTEGER,
    filesize INTEGER,
    position INTEGER CHECK(typeof(position)='integer'),
    inode INTEGER
);"
db_init="
INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JVB', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);
INSERT OR REPLACE INTO state (id, jitsi_component, time, filename, filetime, filesize, position, inode) VALUES (1, 'JICOFO', '1970-01-01 00:00:00.000', '', 0, 0, 0, 0);"
db_flush="
DELETE FROM conferences;
DELETE FROM state;"

help="Usage:\n\t$0 [OPTION]\nOptions:\n\t--create-db|-d - create the database\n\t--flush|-f - flush the tables\n\t--check|-c - check for new data\n\t--parse|-p [-v] - parse the logs [verbosely]"

###

# First we check for requirements
check_requirements() {
    # required programs, anything non-bash - edit as needed
    # deb packages - sqlite3, coreutils(stat,dd)
    local required_programs=("sqlite3" "stat" "dd")
    local requirements_missing=''
    for program in "${required_programs[@]}"; do
        if ! command -v "$program" &> /dev/null; then
            requirements_missing+="$program, "
        fi
    done
    if [[ "$requirements_missing" != '' ]]; then
        requirements_missing=${requirements_missing::-2}
        echo "Error: $requirements_missing - not found. Please install to proceed."
    fi
}
check_requirements

###

# DB functions

# Get the last processed state from the database
get_state() {
    db_get_state=$(printf "$db_get_state_template" "$1")
    sqlite3 "$DB" "$db_get_state"
}

# Update the state database
set_state() {
    local filename=$1
    local filetime=$2
    local filesize=$3
    local position=${4:-0}
    local inode=$5
    local jitsi_component=$6
    db_set_state=$(printf "$db_set_state_template" "$filename" "$filetime" "$filesize" "$position" "$inode" "$jitsi_component")
    sqlite3 "$DB" "$db_set_state"
}

###

# Jitsi Videobridge parsing funstion
jvb_log_parse() {

    local file=$1
    local start_pos=$2
    new_last_pos="$start_pos"

    # Local assoc array for conference events tracking
    declare -A start_times

    # Get size and position for progress tracking
    local total_size=$(stat -c '%s' "$file")
    local processed_lines=0
    local procesed_bytes=0

    # We open the file and start reading from $start_pos bytes
    exec 3<"$file"
    while IFS= read -r line; do
        # save new position (previous plus bytes in current line plus 1 for the new line)
        new_last_pos=$((new_last_pos + ${#line} + 1))

        # increment progress stats
        processed_lines=$((processed_lines + 1))
        processed_bytes=$((processed_bytes + ${#line} + 1))

        # show progress if in verbose mode
        if [[ "$verbose" == true ]]; then
            local percent=$((100 * processed_bytes / total_size))
            echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
        fi

        case $jitsi_component in

            JVB)
                # locate conference starting event
                if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\ Videobridge\.createConference#[0-9]+:\ create_conf,\ id=([a-zA-Z0-9]+) ]]; then
                    timestamp="${BASH_REMATCH[1]}"
                    conferenceId="${BASH_REMATCH[2]}"
                    start_times["$conferenceId"]="$timestamp"

                # locate the corresponding conference ending event
                elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[confId=([a-zA-Z0-9]+)\ .*conf_name=([^ ]+)@.*\]\ Conference\.expire ]]; then
                    end_time="${BASH_REMATCH[1]}"
                    conferenceId="${BASH_REMATCH[2]}"
                    conferenceName="${BASH_REMATCH[3]}"
                    start_time="${start_times["$conferenceId"]}"

                    if [[ -n "$start_time" ]]; then
                        db_insert=$(printf "$db_insert_template" "$jitsi_component" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
                        sqlite3 "$DB" "$db_insert"
                        unset start_times["$conferenceId"]
                    fi
                fi
                ;;

            JICOFO)
                # locate conference starting event
                if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then
                    timestamp="${BASH_REMATCH[1]}"
                    conferenceName="${BASH_REMATCH[2]}"
                    start_times["$conferenceName"]="$timestamp"

                # locate the corresponding conference ending event
                elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then
                    end_time="${BASH_REMATCH[1]}"
                    conferenceName="${BASH_REMATCH[2]}"
                    conferenceId="${BASH_REMATCH[3]}"
                    start_time="${start_times["$conferenceName"]}"

                    if [[ -n "$start_time" ]]; then
                        db_insert=$(printf "$db_insert_template" "$jitsi_component" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
                        sqlite3 "$DB" "$db_insert"
                        unset start_times["$conferenceName"]
                    fi
                fi
                ;;

        esac

    # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
    done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)

    # Close the file descriptor
    exec 3<&-
}

###

# Jicofo parsing funstion
jicofo_log_parse() {

    local file=$1
    local start_pos=$2
    new_last_pos="$start_pos"

    # Local assoc array for conference events tracking
    declare -A start_times

    # Get size and position for progress tracking
    local total_size=$(stat -c '%s' "$file")
    local processed_lines=0
    local procesed_bytes=0

    # We open the file and start reading from $start_pos bytes
    exec 3<"$file"
    while IFS= read -r line; do
        # save new position (previous plus bytes in current line plus 1 for the new line)
        new_last_pos=$((new_last_pos + ${#line} + 1))

        # increment progress stats
        processed_lines=$((processed_lines + 1))
        processed_bytes=$((processed_bytes + ${#line} + 1))

        # show progress if in verbose mode
        if [[ "$verbose" == true ]]; then
            local percent=$((100 * processed_bytes / total_size))
            echo -ne "Processing: $percent% ($processed_lines lines, $processed_bytes bytes) \r"
        fi

        # locate conference starting event
        if [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\]\ JitsiMeetConferenceImpl\.joinTheRoom ]]; then
            timestamp="${BASH_REMATCH[1]}"
            conferenceName="${BASH_REMATCH[2]}"
            start_times["$conferenceName"]="$timestamp"

        # locate the corresponding conference ending event
        elif [[ "$line" =~ ([0-9-]+\ [0-9:.]+)\ [A-Z]+:.*\[room=([^ ]+)@.*\ meeting_id=([a-zA-Z0-9-]+)\]\ JitsiMeetConferenceImpl\.stop ]]; then
            end_time="${BASH_REMATCH[1]}"
            conferenceName="${BASH_REMATCH[2]}"
            conferenceId="${BASH_REMATCH[3]}"
            start_time="${start_times["$conferenceName"]}"

            if [[ -n "$start_time" ]]; then
                db_insert=$(printf "$db_insert_template" "$conferenceName" "$conferenceId" "$start_time" "$end_time")
                sqlite3 "$DB" "$db_insert"
                unset start_times["$conferenceName"]
            fi
        fi
    # We don't use pipe, but process substitution '<(dd...)' to avoid running while loop in subshell and lose the 'new_last_pos' value
    done < <(dd bs=1 skip="$start_pos" <&3 2>/dev/null)

    # Close the file descriptor
    exec 3<&-
}


### FIXME - this is not currently used
# check if and which process is running
is_process_running() {
    pgrep -f "$1" >/dev/null 2>&1
}

# commandline options

while getopts ":dfcpv" opt; do
    case $opt in
        d)
            cmd="--create-db"
            ;;
        f)
            cmd="--flush"
            ;;
        c)
            cmd="--check"
            ;;
        p)
            cmd="--parse"
            ;;
        v)
            verbose=true
            ;;
        \?)
            echo "Invalid option: -$OPTARG" >&2
            echo -e "$help"
            exit 1
            ;;
    esac
done
shift $((OPTIND -1))

case "$cmd" in

    --create-db)
        sqlite3 "$DB" "$db_drop"
        sqlite3 "$DB" "$db_create"
        sqlite3 "$DB" "$db_init"
        echo "Database created."
        exit 0
        ;;

    --flush)
        sqlite3 "$DB" "$db_flush"
        sqlite3 "$DB" "$db_init"
        echo "Tables flushed."
        exit 0
        ;;

    --check)

        # First check if database exists
        if [[ ! -f "$DB" ]]; then
            echo "Database not found. If it's a fresh install, please install the database first."
            exit 1
        fi

        # compare the DB schema to the expected one
        current_db_schema=$(sqlite3 "$DB" .schema)
        if [[ "$current_db_schema" != "$db_create" ]]; then
            echo "The database doesn't match the expected schema. Please check it, and if needed, reinstall it."
            exit 1
        fi

        # Check if log files exist
        jvb_found=false
        jicofo_found=false

        if [[ -f "$JVB_LOGFILE" ]]; then
            jvb_found=true
            jitsi_components+=('JVB')
        fi

        if [[ -f "$JICOFO_LOGFILE" ]]; then
            jicofo_found=true
            jitsi_components+=('JICOFO')
        fi

        # if no logs present, exit
        if [[ "$jvb_found" == false && "$jicofo_found" == false ]]; then
            echo "Neither \"$JVB_PROCESS\" ($JVB_LOGFILE) nor \"$JICOFO_PROCESS\" ($JICOFO_LOGFILE) log files are found."
            exit 1
        else

            # otherwise loop through the found components
            for jitsi_component in "${jitsi_components[@]}"; do

                # Retrieve last log file and position inside it
                IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state "$jitsi_component")

                # Initialize logfile vars
                LOGFILE=$(eval "echo \$${jitsi_component}_LOGFILE")
                ROTATED_LOGFILE="$LOGFILE.1"
                current_inode=$(stat -c '%i' "$LOGFILE")
                current_filetime=$(stat -c '%Y' "$LOGFILE")
                current_size=$(stat -c '%s' "$LOGFILE")

                if [[ "$last_file" == '' || "$last_inode" == 0 ]]; then
                    echo "It looks like a fresh install. You can now run log parsing."
                    exit 0
                fi

                # report
                echo "Last file: $last_file"
                echo "Last filetime: $last_filetime"
                echo "Last inode: $last_inode"
                echo "Last size: $last_size"
                echo "Last processed position: $last_pos"
                echo "Current filetime: $current_filetime"
                echo "Current inode: $current_inode"
                echo "Current size: $current_size"

                if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
                    echo "Log file has rotated."
                else
                    echo "Log file has not rotated."
                fi

                if [[ "$current_filetime" -ne "$last_filetime" || "$current_size" -ne "$last_size" ]]; then
                    echo -e "New lines have been added to the log.\n"
                else
                    echo -e "No new lines in the log.\n"
                fi
            done
        fi

        exit 0
        ;;

    --parse)

        # Check if log files exist
        jvb_found=false
        jicofo_found=false

        if [[ -f "$JVB_LOGFILE" ]]; then
            jvb_found=true
            jitsi_components+=('JVB')
        fi

        if [[ -f "$JICOFO_LOGFILE" ]]; then
            jicofo_found=true
            jitsi_components+=('JICOFO')
        fi

        # if no logs present, exit
        if [[ "$jvb_found" == false && "$jicofo_found" == false ]]; then
            echo "Neither \"$JVB_PROCESS\" ($JVB_LOGFILE) nor \"$JICOFO_PROCESS\" ($JICOFO_LOGFILE) log files are found."
            exit 1
        else

            # otherwise loop through the found components
            for jitsi_component in "${jitsi_components[@]}"; do

                # Retrieve last log file and position inside it
                IFS='|' read -r last_file last_filetime last_size last_pos last_inode <<< $(get_state "$jitsi_component")

                # Initialize logfile vars
                LOGFILE=$(eval "echo \$${jitsi_component}_LOGFILE")
                ROTATED_LOGFILE="$LOGFILE.1"
                last_pos=${last_pos:-0}
                current_inode=$(stat -c '%i' "$LOGFILE")
                current_filetime=$(stat -c '%Y' "$LOGFILE")
                current_size=$(stat -c '%s' "$LOGFILE")

                # Detect if the logfile was rotated (same inode, smaller size - copytruncate in logrotate)
                # parse the rotated log file
                if [[ "$last_inode" == "$current_inode" && "$current_size" -lt "$last_pos" && -f "$ROTATED_LOGFILE" ]]; then
                    echo "Logfile was rotated. Processing the rotated log file: $ROTATED_LOGFILE"
                    jvb_log_parse "$ROTATED_LOGFILE" 0 "$verbose"
                    last_file="$ROTATED_LOGFILE"
                    last_inode=$(stat -c '%i' "$ROTATED_LOGFILE")
                    last_filetime=$(stat -c '%Y' "$ROTATED_LOGFILE")
                    set_state "$last_file" "$last_filetime" "$last_size" "$last_pos" "$last_inode" "$jitsi_component"
                fi

                # parse the current log file
                echo "Processing the current log file: $LOGFILE"
                jvb_log_parse "$LOGFILE" "$last_pos" "$verbose"

                if [[ "$verbose" == true ]]; then
                    echo -e "\nNew last position after parsing: $new_last_pos"
                fi

                # update the state in db
                set_state "$LOGFILE" "$current_filetime" "$current_size" "$new_last_pos" "$current_inode" "$jitsi_component"

            done

        fi

        echo "Data import finished."
        exit 0
        ;;

    *)
        echo -e "$help"
        exit 1
        ;;
esac