Changeset 49942f24


Ignore:
Timestamp:
05/09/14 14:03:19 (15 months ago)
Author:
Kris Moore <kris@…>
Branches:
master, enter/10, releng/10.0.2, releng/10.0.3, releng/10.1, releng/10.1.1, releng/10.1.2
Children:
e0a6855, eaea8ff
Parents:
761148d
Message:

Life-Preserver update:

Add ability to replicate to different hosts from the same zpool.

I.E. your laptop will replicate to one target at work, another one at home,
etc.

Still doing some testing on this, but seems stable so far

Location:
src-sh/lpreserver
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • src-sh/lpreserver/backend/functions.sh

    r1e85620 r49942f24  
    187187  echo "with the correct permissions!" 
    188188 
    189   rem_rep_task "$LDATA" 
     189  rem_rep_task "$LDATA" "$HOST" 
    190190  echo "$LDATA:$TIME:$HOST:$USER:$PORT:$RDATA" >> ${REPCONF} 
    191191 
     
    199199    cronscript="${PROGDIR}/backend/runrep.sh" 
    200200    cLine="$cTime       *       *       *" 
    201     echo -e "$cLine\troot    ${cronscript} ${LDATA}" >> /etc/crontab 
     201    echo -e "$cLine\troot    ${cronscript} ${LDATA} ${HOST}" >> /etc/crontab 
    202202  fi 
    203203} 
     
    205205rem_rep_task() { 
    206206  if [ ! -e "$REPCONF" ] ; then return ; fi 
    207   cat ${REPCONF} | grep -v "^${1}:" > ${REPCONF}.tmp 
     207  cat ${REPCONF} | grep -v "^${1}:.*:${2}:" > ${REPCONF}.tmp 
    208208  mv ${REPCONF}.tmp ${REPCONF} 
    209209 
    210210  # Make sure we remove any old replication entries for this dataset 
    211211  cronscript="${PROGDIR}/backend/runrep.sh" 
    212   cat /etc/crontab | grep -v " $cronscript $1" > /etc/crontab.new 
     212  cat /etc/crontab | grep -v " $cronscript ${1} ${2}" > /etc/crontab.new 
    213213  mv /etc/crontab.new /etc/crontab 
    214214} 
     
    238238  if [ ! -e "$REPCONF" ] ; then return 0; fi 
    239239 
    240   repLine=`cat ${REPCONF} | grep "^${1}:"` 
    241   if [ -z "$repLine" ] ; then return 0; fi 
    242  
    243   # We have a replication task for this dataset, lets check if we need to do it now 
    244   LDATA="$1" 
    245   REPTIME=`echo $repLine | cut -d ':' -f 2` 
    246  
    247   # Export the replication variables we will be using 
    248   export REPHOST=`echo $repLine | cut -d ':' -f 3` 
    249   export REPUSER=`echo $repLine | cut -d ':' -f 4` 
    250   export REPPORT=`echo $repLine | cut -d ':' -f 5` 
    251   export REPRDATA=`echo $repLine | cut -d ':' -f 6` 
    252  
    253   if [ "$2" = "force" ] ; then 
    254      # Ready to do a forced replication 
    255      export DIDREP=1 
    256      echo_log "Starting replication MANUAL task on ${DATASET}: ${REPLOGSEND}" 
    257      queue_msg "`date`: Starting replication MANUAL task on ${DATASET}\n" 
    258      start_rep_task "$LDATA" 
    259      return $? 
    260   fi 
    261  
    262   # If we are checking for a sync task, and the rep isn't marked as sync we can return 
    263   if [ "$2" = "sync" -a "$REPTIME" != "sync" ] ; then return 0; fi 
    264  
    265   # Doing a replication task, check if one is in progress 
    266   export pidFile="${DBDIR}/.reptask-`echo ${LDATA} | sed 's|/|-|g'`" 
    267   if [ -e "${pidFile}" ] ; then 
    268      pgrep -F ${pidFile} >/dev/null 2>/dev/null 
    269      if [ $? -eq 0 ] ; then 
    270         echo_log "Skipped replication on $LDATA, previous replication is still running." 
    271         return 0 
    272      else 
    273         rm ${pidFile} 
    274      fi 
    275   fi 
    276  
    277   # Save this PID 
    278   echo "$$" > ${pidFile} 
    279  
    280   # Is this a sync-task we do at the time of a snapshot? 
    281   if [ "$2" = "sync" -a "$REPTIME" = "sync" ] ; then 
    282      export DIDREP=1 
    283      echo_log "Starting replication SYNC task on ${DATASET}: ${REPLOGSEND}" 
    284      queue_msg "`date`: Starting replication SYNC task on ${DATASET}\n" 
    285      start_rep_task "$LDATA" 
    286      return $? 
    287   else 
    288      # Ready to do a scheduled replication 
    289      export DIDREP=1 
    290      echo_log "Starting replication SCHEDULED task on ${DATASET}: ${REPLOGSEND}" 
    291      queue_msg "`date`: Starting replication SCHEDULED task on ${DATASET}\n" 
    292      start_rep_task "$LDATA" 
    293      return $? 
    294   fi 
     240  ret=0 
     241  for repLine in `cat ${REPCONF} | grep "^${1}:"` 
     242  do 
     243 
     244    # We have a replication task for this dataset, lets check if we need to do it now 
     245    LDATA="$1" 
     246    REPTIME=`echo $repLine | cut -d ':' -f 2` 
     247 
     248    # Export the replication variables we will be using 
     249    export REPHOST=`echo $repLine | cut -d ':' -f 3` 
     250    export REPUSER=`echo $repLine | cut -d ':' -f 4` 
     251    export REPPORT=`echo $repLine | cut -d ':' -f 5` 
     252    export REPRDATA=`echo $repLine | cut -d ':' -f 6` 
     253 
     254    if [ "$2" = "force" ] ; then 
     255       # Ready to do a forced replication 
     256       export DIDREP=1 
     257       echo_log "Starting replication MANUAL task on ${DATASET}: ${REPLOGSEND}" 
     258       queue_msg "`date`: Starting replication MANUAL task on ${DATASET}\n" 
     259       start_rep_task "$LDATA" "$REPHOST" 
     260       if [ $? -ne 0 ] ; then ret=1; fi 
     261       continue 
     262    fi 
     263 
     264    # If we are checking for a sync task, and the rep isn't marked as sync we can continue 
     265    if [ "$2" = "sync" -a "$REPTIME" != "sync" ] ; then continue; fi 
     266 
     267    # Doing a replication task, check if one is in progress 
     268    export pidFile="${DBDIR}/.reptask-`echo ${LDATA} | sed 's|/|-|g'`" 
     269    if [ -e "${pidFile}" ] ; then 
     270       pgrep -F ${pidFile} >/dev/null 2>/dev/null 
     271       if [ $? -eq 0 ] ; then 
     272          echo_log "Skipped replication on $LDATA, previous replication is still running." 
     273          continue 
     274       else 
     275          rm ${pidFile} 
     276       fi 
     277    fi 
     278 
     279    # Save this PID 
     280    echo "$$" > ${pidFile} 
     281 
     282    # Is this a sync-task we do at the time of a snapshot? 
     283    if [ "$2" = "sync" -a "$REPTIME" = "sync" ] ; then 
     284       export DIDREP=1 
     285       echo_log "Starting replication SYNC task on ${DATASET}: ${REPLOGSEND}" 
     286       queue_msg "`date`: Starting replication SYNC task on ${DATASET}\n" 
     287       start_rep_task "$LDATA" 
     288       if [ $? -ne 0 ] ; then ret=1; fi 
     289       continue 
     290    else 
     291       # Ready to do a scheduled replication 
     292       export DIDREP=1 
     293       echo_log "Starting replication SCHEDULED task on ${DATASET}: ${REPLOGSEND}" 
     294       queue_msg "`date`: Starting replication SCHEDULED task on ${DATASET}\n" 
     295       start_rep_task "$LDATA" 
     296       if [ $? -ne 0 ] ; then ret=1; fi 
     297       continue 
     298    fi 
     299  done 
     300  return $ret 
    295301} 
    296302 
     
    300306 
    301307  # Check for the last snapshot marked as replicated already 
    302   lastSEND=`zfs get -d 1 backup:lpreserver ${LDATA} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
     308  lastSEND=`zfs get -d 1 lpreserver:${REPHOST} ${LDATA} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
    303309 
    304310  # Lets get the last snapshot for this dataset 
     
    336342     # Lets mark our new latest snapshot and unmark the last one 
    337343     if [ -n "$lastSEND" ] ; then 
    338        zfs set backup:lpreserver=' ' ${LDATA}@$lastSEND 
     344       zfs set lpreserver:${REPHOST}=' ' ${LDATA}@$lastSEND 
    339345     fi 
    340      zfs set backup:lpreserver=LATEST ${LDATA}@$lastSNAP 
     346     zfs set lpreserver:${REPHOST}=LATEST ${LDATA}@$lastSNAP 
    341347     echo_log "Finished replication task on ${DATASET}" 
    342348     save_rep_props 
     
    387393  for i in `grep "${PROGDIR}/backend/runsnap.sh" /etc/crontab | awk '{print $8}'` 
    388394  do 
    389     echo -e "DATASET - SNAPSHOT - REPLICATION" 
    390     echo "------------------------------------------" 
    391  
    392     lastSEND=`zfs get -d 1 backup:lpreserver ${i} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
    393     lastSNAP=`zfs list -t snapshot -d 1 -H ${i} | tail -1 | awk '{$1=$1}1' OFS=" " | cut -d '@' -f 2 | cut -d ' ' -f 1` 
    394  
    395     if [ -z "$lastSEND" ] ; then lastSEND="NONE"; fi 
    396     if [ -z "$lastSNAP" ] ; then lastSNAP="NONE"; fi 
    397  
    398     echo "$i - $lastSNAP - $lastSEND" 
     395    echo -e "DATASET - TARGET - SNAPSHOT - REPLICATION" 
     396    echo "---------------------------------------------------" 
     397 
     398    # Get some details about this host 
     399    for repLine in `cat ${REPCONF} | grep "^${i}:"` 
     400    do 
     401      REPHOST=`echo $repLine | cut -d ':' -f 3` 
     402 
     403      lastSEND=`zfs get -d 1 lpreserver:${REPHOST} ${i} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
     404      lastSNAP=`zfs list -t snapshot -d 1 -H ${i} | tail -1 | awk '{$1=$1}1' OFS=" " | cut -d '@' -f 2 | cut -d ' ' -f 1` 
     405 
     406      if [ -z "$lastSEND" ] ; then lastSEND="NONE"; fi 
     407      if [ -z "$lastSNAP" ] ; then lastSNAP="NONE"; fi 
     408      echo "$i -> $REPHOST - $lastSNAP - $lastSEND" 
     409    done 
    399410  done 
    400411} 
     
    585596 
    586597  LDATA="$1" 
    587  
    588   repLine=`cat ${REPCONF} | grep "^${LDATA}:"` 
     598  if [ -z "$1" -o -z "$2" ] ; then 
     599     exit_err "Usage: lpreserver replicate init <zpool> <target host>" 
     600  fi 
     601 
     602  repLine=`cat ${REPCONF} | grep "^${LDATA}:.*:${2}:"` 
    589603  if [ -z "$repLine" ] ; then return 0; fi 
    590604  
     
    608622 
    609623  # Now lets mark none of our datasets as replicated 
    610   lastSEND=`zfs get -d 1 backup:lpreserver ${LDATA} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
     624  lastSEND=`zfs get -d 1 lpreserver:${REPHOST} ${LDATA} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
    611625  if [ -n "$lastSEND" ] ; then 
    612      zfs set backup:lpreserver=' ' ${LDATA}@$lastSEND 
     626     zfs set lpreserver:${REPHOST}=' ' ${LDATA}@$lastSEND 
    613627  fi 
    614628 
  • src-sh/lpreserver/backend/runsnap.sh

    r1e85620 r49942f24  
    6464  done 
    6565 
    66   # Get the last replicated snapshot 
    67   lastSEND=`zfs get -d 1 backup:lpreserver ${LDATA} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
    68   if [ -n "$lastSEND" ] ; then 
    69      sec="`echo $lastSEND | cut -d '-' -f 7`" 
    70      min="`echo $lastSEND | cut -d '-' -f 6`" 
    71      hour="`echo $lastSEND | cut -d '-' -f 5`" 
    72      day="`echo $lastSEND | cut -d '-' -f 4`" 
    73      mon="`echo $lastSEND | cut -d '-' -f 3`" 
    74      year="`echo $lastSEND | cut -d '-' -f 2`" 
    75      sendEpoc=`date -j -f "%Y %m %d %H %M %S" "$year $mon $day $hour $min $sec" "+%s"` 
    76      # Check that this replication target is still active 
    77      if [ -e "$REPCONF" ] ; then 
    78        cat ${REPCONF} | grep -q "^${LDATA}:" 
    79        if [ $? -ne 0 ] ; then 
    80           unset lastSEND 
    81           unset sendEpoc 
     66 
     67  # Figure out the oldest snapshot we can prune up to 
     68  for repLine in `cat ${REPCONF} 2>/dev/null | grep "^${LDATA}:"` 
     69  do 
     70    REPHOST=`echo $repLine | cut -d ':' -f 3` 
     71 
     72    # Get the last replicated snapshot 
     73    lastSEND=`zfs get -d 1 lpreserver:${REPHOST} ${LDATA} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
     74    if [ -n "$lastSEND" ] ; then 
     75       sec="`echo $lastSEND | cut -d '-' -f 7`" 
     76       min="`echo $lastSEND | cut -d '-' -f 6`" 
     77       hour="`echo $lastSEND | cut -d '-' -f 5`" 
     78       day="`echo $lastSEND | cut -d '-' -f 4`" 
     79       mon="`echo $lastSEND | cut -d '-' -f 3`" 
     80       year="`echo $lastSEND | cut -d '-' -f 2`" 
     81       newEpoc=`date -j -f "%Y %m %d %H %M %S" "$year $mon $day $hour $min $sec" "+%s"` 
     82       if [ -z "$sendEpoc" ] ; then 
     83          sendEpoc="$newEpoc" 
     84          oldestSEND="$lastSEND" 
     85          continue 
    8286       fi 
    83      fi 
    84   fi 
     87 
     88       # Is this replication older? 
     89       if [ $newEpoc -lt $sendEpoc ] ; then 
     90          sendEpoc="$newEpoc" 
     91          oldestSEND="$lastSEND" 
     92       fi 
     93    fi 
     94  done 
    8595 
    8696  num=0 
     
    92102 
    93103     # If this snapshot is the last one replicated, lets skip pruning it for now 
    94      if [ "$cur" = "$lastSEND" ]; then continue; fi 
     104     if [ "$cur" = "$oldestSEND" ]; then continue; fi 
    95105 
    96106     sec="`echo $snap | cut -d '-' -f 7`" 
  • src-sh/lpreserver/lpreserver

    ra3560cd r49942f24  
    173173Init Options: 
    174174 
    175         init <localdataset/zpool>  
     175        init <localdataset/zpool> <target host> 
    176176 
    177177        Will re-init the remote side of the replication. This can be useful 
     
    181181Remove Options: 
    182182 
    183         remove <dataset> 
     183        remove <dataset> <target host> 
    184184         
    185185        Remove a replication task indicated by <dataset> 
     
    405405}; 
    406406 
     407check_migrate() 
     408{ 
     409   # Check if we need to update the flags on replication to new format 
     410   if [ ! -e "${DBDIR}/.replicationng" -a -e "${REPCONF}" ] ; then 
     411     echo "Updating replication tag system..." 
     412     echo "Please be patient, this is only done once..." 
     413     while read repLine 
     414     do 
     415       local LDATA=`echo $repLine | cut -d ':' -f 1` 
     416       local REPHOST=`echo $repLine | cut -d ':' -f 3` 
     417 
     418       local lastSEND=`zfs get -d 1 backup:lpreserver ${LDATA} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1` 
     419 
     420       if [ -z "$lastSEND" ] ; then continue ; fi 
     421 
     422       # Remove the old flag format and set the new one 
     423       zfs set backup:lpreserver=' ' ${LDATA}@${lastSEND} 
     424       zfs set lpreserver:${REPHOST}=LATEST ${LDATA}@${lastSEND} 
     425 
     426     done < ${REPCONF} 
     427 
     428     # Touch our marker so we don't need to do this again 
     429     touch "${DBDIR}/.replicationng" 
     430   fi 
     431 
     432   # Put other migration functionality as needed here 
     433} 
     434 
    407435# Check if we need to enable the zfs monitor 
    408436enable_watcher 
     437 
     438# See if we need to upgrade flags on replication 
     439check_migrate 
    409440 
    410441# Check what the user wants to do 
     
    506537            case ${1} in 
    507538                add) require_root ; add_rep_task "$2" "$3" "$4" "$5" "$6" "$7" ;; 
    508                 init) require_root ; init_rep_task "$2" ;; 
     539                init) require_root ; init_rep_task "$2" "$3" ;; 
    509540                list) list_rep_task ;; 
    510541                remove) require_root 
Note: See TracChangeset for help on using the changeset viewer.