#!/bin/bash
#Copyright 2005-2024, Lime Technology, Inc.
#License: GPLv2 only

# This is the 'mover' script used for moving files between pools and/or the main unRAID array.
# It is typically invoked via cron.

# First we check if it's valid for this script run: pool use in shfs must be enabled and
# an instance of the script must not already be running.

# Next, check each of the top-level directories (shares) on each pool.
# If, and only if, the 'Use Cache' setting for the share is set to "yes", we use 'find' to
# list the objects (files and directories) of that share directory, moving them to the array.
# Next, we check each of the top-level directories (shares) on each array disk (in sorted order).
# If, and only if, the 'Use Cache' setting for the share is set to "prefer", we use 'find' to
# list the objects (files and directories) of that share directory, moving them to the pool
# associated with the share.

# The script is set up so that hidden directories (i.e., directory names beginning with a '.'
# character) at the topmost level of a pool or an array disk are not moved.  This behavior
# can be turned off by uncommenting the following line:
# shopt -s dotglob

# Files at the top level of a pool or an array disk are never moved.

# The 'find' command generates a list of all files and directories of a share.
# For each file, if the file is not "in use" by any process (as detected by 'fuser' command)
# and not attached to any loop device (as detected by 'losetup' command),
# then the file is moved, and upon success, deleted from the source disk.  If the file already
# exists on the target, it is not moved and the source is not deleted.  All meta-data of moved
# files/directories is preserved: permissions, ownership, extended attributes, and access/modified
# timestamps.

# If an error occurs in copying a file, the partial file, if present, is deleted and the
# operation continues on to the next file.

PIDFILE="/var/run/mover.pid"
CFGPATH="/boot/config/shares"
DEBUGGING=""
MOVERSTATUS="/usr/local/emhttp/state/mover.ini"
GLOBAL_SHARECFG="/boot/config/share.cfg"
MOVER_PROGRESS="disabled"
TOTAL=0
REMAIN=0
SHARE_TOTAL=0
SHARE_REMAIN=0
START_TIME=0
CURRENT_SHARE_STARTED=0
declare -A SHARE_TOTALS SHARE_REMAINS SHARE_START_TIME COUNTED_PATHS

mover_progress_enabled() {
  [[ $MOVER_PROGRESS == "enabled" ]]
}

inivalue() {
  local value="${1//\\/\\\\}"
  value="${value//\"/\\\"}"
  printf '"%s"\n' "$value"
}

tree_size() {
  find "$1" -depth -type f -printf '%s\n' 2>/dev/null | awk '{total += $1} END {print total + 0}'
}

file_size() {
  if [[ -f "$1" ]]; then
    stat -c '%s' -- "$1" 2>/dev/null || echo 0
  else
    echo 0
  fi
}

percent_done() {
  local total="$1"
  local remain="$2"

  if [[ $total -gt 0 ]]; then
    awk -v total="$total" -v remain="$remain" 'BEGIN {printf "%.2f", ((total - remain) / total) * 100}'
  else
    echo 0
  fi
}

throughput_rate() {
  local moved="$1"
  local elapsed="$2"

  if [[ $moved -gt 0 && $elapsed -gt 0 ]]; then
    awk -v moved="$moved" -v elapsed="$elapsed" 'BEGIN {printf "%.0f", moved / elapsed}'
  else
    echo 0
  fi
}

eta_seconds() {
  local remain="$1"
  local rate="$2"

  if [[ $remain -gt 0 && $rate -gt 0 ]]; then
    awk -v remain="$remain" -v rate="$rate" 'BEGIN {printf "%.0f", remain / rate}'
  else
    echo 0
  fi
}

read_pid() {
  [[ -r "$PIDFILE" ]] && tr -d '[:space:]' < "$PIDFILE"
}

is_mover_pid() {
  local pid="$1"
  local cmdline

  [[ $pid =~ ^[0-9]+$ && $pid -gt 0 ]] || return 1
  [[ -r "/proc/$pid/cmdline" ]] || return 1
  cmdline=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null) || return 1
  [[ $cmdline =~ (^|[[:space:]])([^[:space:]]*/)?mover([[:space:]]|$) ]]
}

cleanup() {
  local pid

  pid=$(read_pid)
  if [[ $pid == "$$" ]]; then
    rm -f -- "$PIDFILE" "$MOVERSTATUS" "${MOVERSTATUS}.$$"
  fi
}

acquire_pidfile() {
  local pid

  if [[ -f "$PIDFILE" ]]; then
    pid=$(read_pid)
    if is_mover_pid "$pid"; then
      echo "mover: already running" >&2
      exit 1
    fi
    rm -f -- "$PIDFILE"
  fi

  if ! (set -o noclobber; printf '%s\n' "$$" > "$PIDFILE") 2>/dev/null; then
    pid=$(read_pid)
    if is_mover_pid "$pid"; then
      echo "mover: already running" >&2
      exit 1
    fi
    rm -f -- "$PIDFILE"
    if ! (set -o noclobber; printf '%s\n' "$$" > "$PIDFILE") 2>/dev/null; then
      echo "mover: already running" >&2
      exit 1
    fi
  fi
}

writestatus() {
  mover_progress_enabled || return

  local file="$1"
  local action="$2"
  local filepercent="${3:-0}"
  local progress_title="${4:-Share Progress:}"
  local tmp="${MOVERSTATUS}.$$"
  local now elapsed moved rate eta share_elapsed share_moved share_rate share_eta

  now=$(date +%s)
  elapsed=$((START_TIME > 0 && now > START_TIME ? now - START_TIME : 0))
  moved=$((TOTAL > REMAIN ? TOTAL - REMAIN : 0))
  rate=$(throughput_rate "$moved" "$elapsed")
  eta=$(eta_seconds "$REMAIN" "$rate")
  share_elapsed=$((CURRENT_SHARE_STARTED > 0 && now > CURRENT_SHARE_STARTED ? now - CURRENT_SHARE_STARTED : 0))
  share_moved=$((SHARE_TOTAL > SHARE_REMAIN ? SHARE_TOTAL - SHARE_REMAIN : 0))
  share_rate=$(throughput_rate "$share_moved" "$share_elapsed")
  share_eta=$(eta_seconds "$SHARE_REMAIN" "$share_rate")

  {
    echo "TotalSize=${TOTAL}"
    echo "RemainSize=${REMAIN}"
    echo "Throughput=${rate}"
    echo "ETA=${eta}"
    echo "ShareTotal=${SHARE_TOTAL}"
    echo "ShareRemain=${SHARE_REMAIN}"
    echo "ShareThroughput=${share_rate}"
    echo "ShareETA=${share_eta}"
    echo "MoverPid=$$"
    printf 'File=%s\n' "$(inivalue "$file")"
    printf 'Action=%s\n' "$(inivalue "$action")"
    echo "Filepercent=${filepercent}"
    echo "Showlines=5"
    echo "Selectlines=1,2,3,4,5"
    echo "movertitle1=File being processed:"
    echo "movertitle2=Action:"
    echo "movertitle3=${progress_title}"
    echo "movertitle4=Overall Progress:"
    echo "movertitle5=Estimated Completion:"
  } > "$tmp"
  if ! mv -f -- "$tmp" "$MOVERSTATUS"; then
    rm -f -- "$tmp" "$MOVERSTATUS"
    return 1
  fi
}

count_path() {
  local path="$1"
  local normalized="${path%/}"
  local size

  [[ -n ${COUNTED_PATHS[$normalized]+x} ]] && return
  COUNTED_PATHS["$normalized"]=1
  size=$(tree_size "$normalized")
  TOTAL=$((TOTAL + size))
  REMAIN=$((REMAIN + size))
  SHARE_TOTALS["$SHARE"]=$((${SHARE_TOTALS["$SHARE"]:-0} + size))
  SHARE_REMAINS["$SHARE"]=$((${SHARE_REMAINS["$SHARE"]:-0} + size))
}

prepare_share_progress() {
  local share_key="$SHARE"

  if [[ -z ${SHARE_START_TIME[$share_key]} ]]; then
    SHARE_START_TIME[$share_key]=$(date +%s)
  fi
  CURRENT_SHARE_STARTED=${SHARE_START_TIME[$share_key]}
  SHARE_TOTAL=${SHARE_TOTALS["$SHARE"]:-0}
  SHARE_REMAIN=${SHARE_REMAINS["$SHARE"]:-0}
}

decrement_progress() {
  local size="$1"

  [[ $size -le 0 ]] && return
  REMAIN=$((REMAIN > size ? REMAIN - size : 0))
  SHARE_REMAINS["$SHARE"]=$((${SHARE_REMAINS["$SHARE"]:-0} > size ? ${SHARE_REMAINS["$SHARE"]:-0} - size : 0))
  SHARE_REMAIN=${SHARE_REMAINS["$SHARE"]:-0}
}

move() {
  local path
  local size
  local action
  local cleanup_action
  local empty_disk="${2:-}"
  local progress=0
  local progress_title="Share Progress:"

  if mover_progress_enabled; then
    progress=1
    prepare_share_progress
    if [[ -n $empty_disk ]]; then
      action="Emptying ${empty_disk}"
      cleanup_action="Emptying ${empty_disk} (cleaning up)"
      progress_title="Disk Progress:"
    else
      action="Moving share ${SHARE}"
      cleanup_action="Cleaning up share ${SHARE}"
    fi
  fi
  while IFS= read -r path; do
    if [[ $progress -eq 1 ]]; then
      size=$(file_size "$path")
      writestatus "$path" "$action" "$(percent_done "$SHARE_TOTAL" "$SHARE_REMAIN")" "$progress_title"
    fi
    printf '%s\n' "$path" | /usr/libexec/unraid/move $DEBUGGING
    if [[ $progress -eq 1 && $size -gt 0 && ! -e "$path" ]]; then
      decrement_progress "$size"
    fi
  done < <(find "$1" -depth 2>/dev/null)

  # second pass to clean up leftover empty directories
  while IFS= read -r path; do
    if [[ $progress -eq 1 ]]; then
      writestatus "$path" "$cleanup_action" "$(percent_done "$SHARE_TOTAL" "$SHARE_REMAIN")" "$progress_title"
    fi
    printf '%s\n' "$path" | /usr/libexec/unraid/move $DEBUGGING
  done < <(find "$1" -depth -type d 2>/dev/null)
}

calculate_totals() {
  local sharecfg sharepath disk emptying
  local -a disks arr

  TOTAL=0
  REMAIN=0
  SHARE_TOTALS=()
  SHARE_REMAINS=()
  SHARE_START_TIME=()
  COUNTED_PATHS=()

  for sharecfg in $CFGPATH/*.cfg ; do
    SHARE=$(basename "$sharecfg" .cfg)
    shareUseCache=""
    shareCachePool=""
    shareCachePool2=""
    source <(fromdos < "$sharecfg")

    if [[ $shareUseCache = yes && -n $shareCachePool && -d "/mnt/$shareCachePool/$SHARE" ]]; then
      if [[ -n $shareCachePool2 ]]; then
        if [[ -d "/mnt/$shareCachePool2" ]]; then
          count_path "/mnt/$shareCachePool/$SHARE"
        fi
      else
        disks=(/mnt/disk[1-9]/ /mnt/disk[1-9][0-9]/)
        if [[ ${#disks[@]} -gt 0 ]]; then
          count_path "/mnt/$shareCachePool/$SHARE"
        fi
      fi
    fi

    if [[ $shareUseCache = prefer ]]; then
      if [[ -n $shareCachePool2 ]]; then
        if [[ -d "/mnt/$shareCachePool2/$SHARE" && -n $shareCachePool && -d "/mnt/$shareCachePool" ]]; then
          count_path "/mnt/$shareCachePool2/$SHARE"
        fi
      else
        if [[ -n $shareCachePool && -d "/mnt/$shareCachePool" ]]; then
          for sharepath in /mnt/disk[1-9]/"$SHARE"/ /mnt/disk[1-9][0-9]/"$SHARE"/ ; do
            count_path "$sharepath"
          done
        fi
      fi
    fi
  done

  emptying=$(grep '^shareUserEmptying=' /boot/config/share.cfg | cut -d'"' -f2)
  IFS=',' read -ra arr <<< "$emptying"
  if [[ -n $emptying ]]; then
    for disk in "${arr[@]}"; do
      disk="${disk//[[:space:]]/}"
      [[ $disk =~ ^disk[1-9][0-9]*$ ]] || continue
      for sharepath in "/mnt/$disk"/* ; do
        SHARE=$(basename "$sharepath")
        if [[ -d "$sharepath" && -f "$CFGPATH/$SHARE.cfg" ]]; then
          count_path "$sharepath"
        fi
      done
    done
  fi
}

start() {
  trap cleanup EXIT
  trap 'exit 143' HUP INT TERM
  acquire_pidfile
  echo "mover: started" >&2
  MOVER_PROGRESS=$(grep '^shareMoverProgress=' "$GLOBAL_SHARECFG" 2>/dev/null | cut -d'"' -f2)
  MOVER_PROGRESS=${MOVER_PROGRESS:-disabled}
  rm -f -- "$MOVERSTATUS"

  shopt -s nullglob
  if mover_progress_enabled; then
    START_TIME=$(date +%s)
    writestatus "" "Calculating mover totals" 0
    calculate_totals
    writestatus "" "Mover totals calculated" 0
    sleep 2
  fi

  for SHARECFG in $CFGPATH/*.cfg ; do
    SHARE=$(basename "$SHARECFG" .cfg)
    shareUseCache=""
    shareCachePool=""
    shareCachePool2=""
    source <(fromdos < "$SHARECFG")

    # maybe move from primary to secondary
    if [[ $shareUseCache = yes && -n $shareCachePool && -d "/mnt/$shareCachePool/$SHARE" ]]; then
      if [[ -n $shareCachePool2 ]]; then
        # secondary is a pool
        if [[ -d "/mnt/$shareCachePool2" ]]; then
          move "/mnt/$shareCachePool/$SHARE"
        fi
      else
        # secondary is the unRAID array, ensure at least one data disk is present
        disks=(/mnt/disk[1-9]/ /mnt/disk[1-9][0-9]/)
        if [[ ${#disks[@]} -gt 0 ]]; then
          move "/mnt/$shareCachePool/$SHARE"
        fi
      fi
    fi

    # maybe move from secondary to primary
    if [[ $shareUseCache = prefer ]]; then
      if [[ -n $shareCachePool2 ]]; then
        # secondary is a pool (primary is also a pool)
        if [[ -d "/mnt/$shareCachePool2/$SHARE" && -n $shareCachePool && -d "/mnt/$shareCachePool" ]]; then
          move "/mnt/$shareCachePool2/$SHARE"
        fi
      else
        # secondary is the unRAID array
        if [[ -n $shareCachePool && -d "/mnt/$shareCachePool" ]]; then
          for SHAREPATH in /mnt/disk[1-9]/"$SHARE"/ /mnt/disk[1-9][0-9]/"$SHARE"/ ; do
            move "$SHAREPATH"
          done
        fi
      fi
    fi
  done

  # maybe empty an array disk
  EMPTYING=$(grep '^shareUserEmptying=' /boot/config/share.cfg | cut -d'"' -f2)
  IFS=',' read -ra arr <<< "$EMPTYING"
  if [[ -n $EMPTYING ]]; then
    for DISK in "${arr[@]}"; do
      DISK="${DISK//[[:space:]]/}"
      [[ $DISK =~ ^disk[1-9][0-9]*$ ]] || continue
      # we can only empty share directories
      for SHAREPATH in "/mnt/$DISK"/* ; do
        SHARE=$(basename "$SHAREPATH")
        if [[ -d "$SHAREPATH" && -f "$CFGPATH/$SHARE.cfg" ]]; then
          move "$SHAREPATH" "$DISK"
        fi
      done

      # output list of files which could not be moved
      # use 'find' in case huge number of files left in /mnt/$DISK
      count=$(find "/mnt/$DISK" -mindepth 1 | wc -l)
      if [ "$count" -gt 0 ]; then
        find "/mnt/$DISK" -mindepth 1 -depth -printf 'move: %p Not moved\n' | head -n 100
        if [ "$count" -gt 100 ]; then
          echo "[output truncated to first 100 entries]"
          writestatus "" "Mover output truncated" "$(percent_done "$SHARE_TOTAL" "$SHARE_REMAIN")"
        fi
      fi
    done
  fi

  rm -f -- "$PIDFILE" "$MOVERSTATUS"
  echo "mover: finished" >&2
}

stop() {
  local pid

  if [[ ! -f "$PIDFILE" ]]; then
    echo "mover: not running"
    return 0
  fi
  pid=$(read_pid)
  if ! is_mover_pid "$pid"; then
    rm -f -- "$PIDFILE" "$MOVERSTATUS"
    echo "mover: not running"
    return 0
  fi

  kill -TERM "$pid" 2>/dev/null || true
  for _ in {1..20}; do
    is_mover_pid "$pid" || break
    sleep 0.1
  done
  if is_mover_pid "$pid"; then
    kill -KILL "$pid" 2>/dev/null || true
  fi
  rm -f -- "$PIDFILE" "$MOVERSTATUS"
  echo "mover: stopped"
}

# display usage and then exit
usage() {
    echo "Usage: $0 start|stop|status"
    exit 1
}

if [ "$#" -ne 1 ]; then
    usage
    exit 1
fi

case $1 in
start)
  start
  ;;
stop)
  stop
  ;;
status)
  [[ -f "$PIDFILE" ]] && is_mover_pid "$(read_pid)"
  ;;
*)
  usage
  ;;
esac
