#!/bin/bash
#
# Cellular modem connection script based on ModemManager
# Copyright (C) Signalytic
# 2024-03-07 WA
#
# Tests:
#   basic functional test
#   disconnect modem - reconnect modem
#   remove sim - reinsert sim (bearer must be deleted)
#

CONFDIR="/etc/signalytic"
NODCONFDIR=$(grep nodeconfig "${CONFDIR}/blockgraph" | cut -d= -f2 | xargs)
ROOT_DIR="${NODCONFDIR}/.."
STATUS_DIR=${ROOT_DIR}/status
mkdir -p $STATUS_DIR
STATUS_FILE=$STATUS_DIR/modem

HARDRESET_TIME_TO_REBOOT=129600   # 36 hours
HARDRESET_BOOT_HOUR_START=0
HARDRESET_BOOT_HOUR_END=5

# Reset escalation intervals - total delay is cumulative, so AT command reset is
# 1.5 hours after first reset call.
RESET_LEVEL_1_INTERVAL=1800      # 30 min - service restart
RESET_LEVEL_2_INTERVAL=600       # 10 min - AT command reset
RESET_LEVEL_3_INTERVAL=600       # 10 min - USB reset
RESET_LEVEL_4_INTERVAL=600       # 10 min - kernel module reset

CONF_CELLMODEM=${CONFDIR}/cellmodem
APN=$(grep apn ${CONF_CELLMODEM} | cut -d= -f2 | xargs)

# Should only ever be one modem - may be empty if modem is disabled:
MODEM_PATH=$(mmcli -L -K|grep "modem-list.value\[1\]"|cut -d: -f2|xargs)

is_running() {
  pidof -q $1 &> /dev/null
}

status() {
  echo "${1}" > $STATUS_FILE
}

wait_for_ping() {
  timeout=$1
  interval=$2
  device=$3
  count=0
  while true; do
    count=$(( count + interval ))
    # Try to reach Google's DNS server
    if ping -I ${device} -W ${interval} -c 1 8.8.8.8 &> /dev/null; then
      return 0
    elif [ $count -ge $timeout ]; then
      echo "No network connection"
      return 1
    fi
  done
}

wait_for_device() {
  timeout=$1
  interval=$2
  count=0
  echo -n "Waiting for modem to appear... "
  while true; do
    count=$(( count + interval ))
    list_count=$(mmcli -LK|grep "modem-list.length"|cut -d: -f2|xargs)
    # Label changes when there are no modems, giving empty string:
    if [ "$list_count" ] && [ "$list_count" -ne "0" ]; then
      MODEM_PATH=$(mmcli -L -K|grep "modem-list.value\[1\]"|cut -d: -f2|xargs)
      echo "OK"
      return 0
    elif [ $count -ge $timeout ]; then
      echo "FAIL"
      return 1
    fi
    sleep ${interval}
  done
}

find_by_product(){
  syspath=$(grep -il "^$product" /sys/bus/usb/devices/*/product)
  if [ "$syspath" ]; then
    dirname "$syspath"
  else
    echo ""
  fi
}

reset1_service_restart() {
  echo "  Attempting ModemManager service restart..."
  bgsystemlog -s NETWORK -l WARNING -m 'RESET: Cellmodem service restart'
  # Clear any existing bearers first
  if [ "$MODEM_PATH" ]; then
    mmcli -m ${MODEM_PATH} --simple-disconnect 2>/dev/null
  fi
  # Stop and restart ModemManager with cache clear
  systemctl stop ModemManager
  sleep 3
  # Clear ModemManager cache if it exists
  rm -f /var/lib/ModemManager/connection-history 2>/dev/null
  systemctl start ModemManager
  sleep 10  # Give time for service to initialize
  echo "  Service restart completed"
}

reset2_at_command() {
  echo "  Attempting AT command reset..."
  bgsystemlog -s NETWORK -l WARNING -m 'RESET: Cellmodem AT command reset'
  # Find the AT command interface (usually ttyUSB2 for EC25)
  tty_list=("$@")
  for tty in "${tty_list[@]}"; do
    echo "  Trying AT reset on $tty"
    # Set function to minimum (disable radio)
    timeout 5 sh -c "echo -e 'AT+CFUN=0\r' > /dev/$tty" 2>/dev/null
    sleep 3
    # Reset to full functionality
    timeout 5 sh -c "echo -e 'AT+CFUN=1,1\r' > /dev/$tty" 2>/dev/null
    sleep 2
    # Try to get modem info to verify it's responding
    if timeout 5 sh -c "echo -e 'AT+CGSN\r' > /dev/$tty" 2>/dev/null; then
      echo "  AT command reset appears successful on $tty"
      return 0
    fi
  done
  echo "  AT command reset failed - no responsive interface found"
  return 1
}

reset3_usb_subsystem() {
  echo "  Attempting USB subsystem reset..."
  bgsystemlog -s NETWORK -l WARNING -m 'RESET: Cellmodem USB subsystem reset'
  product="EC25-EUX"
  # Find the USB device path for the modem
  usb_modem_syspath=$(find_by_product "$product")
  if [ "$usb_modem_syspath" ]; then
    echo "Found modem at $usb_modem_syspath"
    # Get the USB device ID
    usb_device=$(basename "$usb_modem_syspath")
    # Unbind and rebind the USB device
    echo "$usb_device" > /sys/bus/usb/drivers/usb/unbind 2>/dev/null
    sleep 3
    echo "$usb_device" > /sys/bus/usb/drivers/usb/bind 2>/dev/null
    sleep 5
    echo "  USB subsystem reset completed for device $usb_device"
    return 0
  else
    echo "  Could not find USB device path for $product"
    return 1
  fi
}

reset4_kernel_module() {
  echo "  Attempting kernel module reset..."
  bgsystemlog -s NETWORK -l WARNING -m 'RESET: Cellmodem kernel module reset'
  # Stop ModemManager first
  systemctl stop ModemManager
  sleep 2
  # Remove modem-related kernel modules
  modprobe -r qmi_wwan cdc_wdm usb_wwan option 2>/dev/null
  modprobe -r cdc_mbim 2>/dev/null
  sleep 3
  # Reload modules
  modprobe option
  modprobe usb_wwan
  modprobe cdc_wdm
  modprobe qmi_wwan
  modprobe cdc_mbim
  sleep 5
  # Restart ModemManager
  systemctl start ModemManager
  sleep 5
  echo "  Kernel module reset completed"
}

reset_alt_usb_power_cycle() {
  product="EC25-EUX"
  echo "  Attempting USB power cycle reset of ${product}..."
  systemctl stop ModemManager
  fallback_reset=1
  usb_modem=$(usbreset 2>/dev/null|grep '^\s\+Number'|grep "$product"|xargs|cut -d' ' -f4)
  if [ "$usb_modem" ]; then
    # modem is present but hung, so we will try to power-cycle the interface
    usb_modem_syspath=$(find_by_product "$product")
    if [ "$usb_modem_syspath" ]; then
      echo "0" > "$usb_modem_syspath/power/autosuspend_delay_ms"
      echo "auto" > "$usb_modem_syspath/power/control"
      sleep 3
      echo "2000" > "$usb_modem_syspath/power/autosuspend_delay_ms"
      echo "on" > "$usb_modem_syspath/power/control"
      sleep 2
      usbreset "$usb_modem" 2>/dev/null
      fallback_reset=0
      bgsystemlog -s NETWORK -l WARNING -m 'Cellmodem USB power cycle executed'
    else
      echo "  ERROR: syspath for ${product} was not found"
    fi
  fi
  if [ "$fallback_reset" -eq 1 ]; then
    # modem not detected or unable to reset, so we reset all USB hubs to trigger
    # bus enumeration:
    echo "  WARNING: resetting all USB buses as fallback"
    while read -r line; do
      grep -qi hub <<< $line && usbreset $(echo $line|xargs|cut -d' ' -f4) 2>/dev/null
    done < <(usbreset 2>/dev/null|grep '^\s\+Number')
    bgsystemlog -s NETWORK -l WARNING -m 'Cellmodem USB hub reset fallback executed'
  fi
  sleep 2
  systemctl start ModemManager
  echo "  INFO: USB power cycle RESET completed for ${product}"
}

modem_reset() {
  product="EC25-EUX"
  # Parse existing status to determine reset history and escalation level
  old_status=$(cat ${STATUS_FILE} 2>/dev/null || echo "")
  time_since_first=0                          # time since first reset attempt
  time_since_last="$HARDRESET_TIME_TO_REBOOT" # time since last reset attempt
  time_now=$(date +%s)
  reset_level=1                               # start with service restart
  
  if [ "${old_status:0:9}" == "hardreset" ]; then
    time1=$(echo "$old_status" | cut -d' ' -f2)
    time2=$(echo "$old_status" | cut -d' ' -f3)
    reset_level=$(echo "$old_status" | cut -d' ' -f4 2>/dev/null || echo "1")
    time_since_first=$(( time_now - time1 ))
    time_since_last=$(( time_now - time2 ))
    echo "Previous reset status: ${old_status}"
  else
    bgsystemlog -s NETWORK -l WARNING -m 'Cellmodem reset initiated'
    time1="$time_now"
  fi
  
  echo "  time since first attempt: ${time_since_first}s"
  echo "  time since last attempt: ${time_since_last}s"
  echo "  current reset level: ${reset_level}"
  
  # If all escalation levels have failed over extended time: reboot at night:
  if [ $time_since_first -ge $HARDRESET_TIME_TO_REBOOT ]; then
    if [ $(date +%H) -ge $HARDRESET_BOOT_HOUR_START ] && [ $(date +%H) -lt $HARDRESET_BOOT_HOUR_END ]; then
      status "disconnected"
      [ "$MODEM_PATH" ] || MODEM_PATH=$(mmcli -L -K|grep "modem-list.value\[1\]"|cut -d: -f2|xargs)
      mmcli -m ${MODEM_PATH} -K     # dump state in log file, for debugging
      bgsystemlog -s NETWORK -l ERROR -m "Cellmodem recovery failed after ${time_since_first}s - REBOOT"
      echo "WARNING: rebooting to restore cellmodem connection after all escalation levels failed"
      reboot
      sleep 10
    fi
    return 0
  fi
  
  # Determine which reset level to execute based on time intervals
  execute_reset=0
  case $reset_level in
    1)
      if [ "$time_since_last" -gt $RESET_LEVEL_1_INTERVAL ]; then
        echo "Executing Reset Level 1:"
        reset1_service_restart
        execute_reset=1
        reset_level=2
      fi
      ;;
    2)
      if [ "$time_since_last" -gt $RESET_LEVEL_2_INTERVAL ]; then
        echo "Executing Reset Level 2:"
        tty_list=()
        if [ $MODEM_PATH ]; then
          tty_list=( $(mmcli -m ${MODEM_PATH} -K|grep 'modem.generic.ports.value'|grep '(at)'|cut -d: -f2|awk '{$1=$1};1'|cut -d' ' -f1) )
        fi
        if [ ! "${tty_list}" ]; then
          tty_list=( $(basename -a $(ls -1 /dev/ttyUSB*)) )
        fi
        reset2_at_command "${tty_list[@]}"
        execute_reset=1
        reset_level=3
      fi
      ;;
    3)
      if [ "$time_since_last" -gt $RESET_LEVEL_3_INTERVAL ]; then
        echo "Executing Reset Level 3:"
        if ! reset3_usb_subsystem; then
          # Fallback to old USB reset method
          echo "Falling back to USB power cycle reset"
          reset_alt_usb_power_cycle
        fi
        execute_reset=1
        reset_level=4
      fi
      ;;
    4)
      if [ "$time_since_last" -gt $RESET_LEVEL_4_INTERVAL ]; then
        echo "Executing Reset Level 4:"
        reset4_kernel_module
        execute_reset=1
        reset_level=1     # return to first
      fi
      ;;
    *)
      # Level 5 and above: wait for reboot window
      echo "All escalation levels exhausted, waiting for reboot window"
      ;;
  esac
  if [ $execute_reset -eq 1 ]; then
    status "hardreset $time1 $time_now $reset_level"
    # Give the reset some time to take effect
    sleep 5
  fi
  return 0
}

if [ $(id -u) -ne 0 ]; then
  echo "Root permission required" 1>&2
  exit 1
fi

# Check if we are still running:
if is_running signalytic-ppp; then
  echo "signalytic-ppp is already running (overrun)"
  exit 1
fi

# Check if we are (still) connected:
BEARER_PATH=""
if [ $MODEM_PATH ]; then
  while read -r line; do
    grep -q "modem.generic.state\s" <<< $line && modem_state=$(echo "$line"|cut -d: -f2|xargs)
    grep -q "modem.generic.bearers.length\s" <<< $line && bearers_count=$(echo "$line"|cut -d: -f2|xargs)
    grep -q "modem.generic.bearers.value\[${bearers_count}" <<< $line && BEARER_PATH=$(echo "$line"|cut -d: -f2|xargs)
  done < <(mmcli -m ${MODEM_PATH} -K)
fi

if [ $BEARER_PATH ]; then
  while read -r line; do
    grep -q "bearer.status.connected\s" <<< $line && bearer_status=$(echo "$line"|cut -d: -f2|xargs)
    grep -q "bearer.status.interface\s" <<< $line && bearer_interface=$(echo "$line"|cut -d: -f2|xargs)
  done < <(mmcli -b ${BEARER_PATH} -K)
  if [ "$bearer_status" != "yes" ]; then
    echo "WARNING: bearer down, cleaning up stale bearer"
    mmcli -m ${MODEM_PATH} --simple-disconnect 2>/dev/null
  elif ! wait_for_ping 30 2 ${bearer_interface}; then
    echo "WARNING: ping fail on ${bearer_interface}"
  else
    # Connection is good - this will set status to "connected" if all is well.
    status "$modem_state"
    exit 0
  fi
fi

# ------------------------------------------------------------------------------
# Something is not working. Reset the connection and try to reconnect:
echo
date --utc --iso-8601='seconds'
echo "Starting connection attempt."
echo "APN: ${APN}"

# Get modem path with extended timeout:
if ! wait_for_device 120 5; then
  echo "WARNING: No modem path, attempting reset:"
  # This also sets status:
  modem_reset
  exit 2
fi
echo "Modem path: ${MODEM_PATH}"

bearers_count=0
mmcli -m ${MODEM_PATH} --simple-connect="apn=${APN},ip-type=ipv4v6"
while read -r line; do
  grep -q "modem.generic.state\s" <<< $line && modem_state=$(echo "$line"|cut -d: -f2|xargs)
  grep -q "modem.generic.bearers.length\s" <<< $line && bearers_count=$(echo "$line"|cut -d: -f2|xargs)
  grep -q "modem.generic.bearers.value\[${bearers_count}" <<< $line && BEARER_PATH=$(echo "$line"|cut -d: -f2|xargs)
done < <(mmcli -m ${MODEM_PATH} -K)

# Set up network interface
if [ "$bearers_count" -eq "0" ]; then
  echo "ERROR: no bearer, attempting reset:"
  modem_reset
  exit 3
fi
echo "Bearer path: ${BEARER_PATH}"

# Basic assessment of bearer state:
while read -r line; do
  grep -q "bearer.ipv4-config.method\s" <<< $line && config_method=$(echo "$line"|cut -d: -f2|xargs)
  grep -q "bearer.ipv6-config.method\s" <<< $line && config_method_ipv6=$(echo "$line"|cut -d: -f2|xargs)
  grep -q "bearer.status.connected\s" <<< $line && bearer_status=$(echo "$line"|cut -d: -f2|xargs)
  grep -q "bearer.status.connection-error.name\s" <<< $line && bearer_error_name=$(echo "$line"|cut -d: -f2|xargs)
  grep -q "bearer.status.connection-error.message\s" <<< $line && bearer_error_message=$(echo "$line"|cut -d: -f2|xargs)
  grep -q "bearer.status.interface" <<< $line && INTERFACE=$(echo "$line"|cut -d: -f2|xargs)
done < <(mmcli -b ${BEARER_PATH} -K)

if [ "$bearer_status" != "yes" ] || [ "$INTERFACE" == "--" ]; then
  echo "ERROR: bearer not connected"
  echo "  ${bearer_error_name}"
  echo "  ${bearer_error_message}"
  modem_reset
  exit 4
fi

ip_mode="ipv4"
if [ "$config_method" == "--" ]; then
  config_method="$config_method_ipv6"
  ip_mode="ipv6"
fi
if [ "$config_method" == "--" ]; then
  echo "ERROR: no IP networking"
  # delete all bearers to make new connection next time
  mmcli -m ${MODEM_PATH} --simple-disconnect
  status "disconnected"
  exit 4
fi

dns_num=0
if [ "$config_method" == "static" ]; then
  while read -r line; do
    grep -q "bearer.${ip_mode}-config.address" <<< $line && IP_ADDRESS=$(echo "$line"|cut -d: -f2|xargs)
    grep -q "bearer.${ip_mode}-config.prefix" <<< $line && IP_PREFIX=$(echo "$line"|cut -d: -f2|xargs)
    grep -q "bearer.${ip_mode}-config.gateway" <<< $line && IP_GATEWAY=$(echo "$line"|cut -d: -f2|xargs)
    grep -q "bearer.${ip_mode}-config.mtu" <<< $line && IP_MTU=$(echo "$line"|cut -d: -f2|xargs)
    grep -q "bearer.${ip_mode}-config.dns.length" <<< $line && dns_num=$(echo "$line"|cut -d: -f2|xargs)
    for (( i = 1; i <= dns_num; i++ )); do
      grep -q "bearer.${ip_mode}-config.dns.value\[${i}" <<< $line && IP_DNS[$i]=$(echo "$line"|cut -d: -f2|xargs)
    done
  done < <(mmcli -b ${BEARER_PATH} -K)
fi

# configure network interface
IP_OPTS=""
if [ "$ip_mode" == "ipv6" ]; then
  IP_OPTS="-6"
fi
ip addr flush dev ${INTERFACE}
ip -6 addr flush dev ${INTERFACE}
ip ${IP_OPTS} link set ${INTERFACE} up
ip ${IP_OPTS} addr add ${IP_ADDRESS}/${IP_PREFIX} dev ${INTERFACE}
# drop any routes that may be problematic
route del default &> /dev/null
# add new
ip ${IP_OPTS} route add default via ${IP_GATEWAY} dev ${INTERFACE}
ip ${IP_OPTS} link set ${INTERFACE} arp off
ip ${IP_OPTS} link set ${INTERFACE} mtu ${IP_MTU} 

# DNS
sed '/## temporary DNS/q' /etc/resolv.conf > /tmp/resolv.conf-wlan
echo "## temporary DNS" >> /tmp/resolv.conf-wlan
for ns in "${IP_DNS[@]}"; do
  echo "nameserver $ns" >> /tmp/resolv.conf-wlan
done
mv /tmp/resolv.conf-wlan /etc/resolv.conf

if ! wait_for_ping 60 2 ${INTERFACE}; then
  # Note: we cannot simply assumed the connection is good. If we got here then
  # we have not called modem_reset before, and the ping test fails.
  echo "WARNING: ping fail after reinit on ${INTERFACE}"
  modem_reset
  exit 5
else
  # Connection is good - this will set status to "connected" if all is well.
  status "$modem_state"
fi

exit 0

