d0d307
#!/bin/sh
d0d307
d0d307
# This program is free software; you can redistribute it and/or modify
d0d307
# it under the terms of the GNU General Public License as published by
d0d307
# the Free Software Foundation; either version 2 of the License, or
d0d307
# (at your option) any later version.
d0d307
#
d0d307
# This program is distributed in the hope that it will be useful,
d0d307
# but WITHOUT ANY WARRANTY; without even the implied warranty of
d0d307
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
d0d307
# GNU General Public License for more details.
d0d307
#
d0d307
# You should have received a copy of the GNU General Public License
d0d307
# along with this program; if not, write to the Free Software
d0d307
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
d0d307
d0d307
# This script converts all db files of a cyrus installation from their
d0d307
# existing format to the format required by the current installation.
d0d307
# The format of current db files is determined using the 'file' command
d0d307
# with a magic file added for skiplist db, the new format is read from
d0d307
# a config file usually in /usr/share/cyrus-imapd/rpm/db.cfg, which is
d0d307
# created while compiling. After converting, the db.cfg file is
d0d307
# copied to a cache file usually at /var/lib/imap/rpm/db.cfg.cache to
d0d307
# allow bypassing this converting script if both files are identical.
d0d307
# While this is a bit less secure, it may be useful on big server where
d0d307
# db converting is done automatically.
d0d307
#
d0d307
# This script can safely be run as root, it will reexec itself as user
d0d307
# cyrus if needed.
d0d307
#
d0d307
# author: Simon Matter, Invoca Systems <simon.matter@invoca.ch>
d0d307
d0d307
# changelog
d0d307
# v1.0.1, Oct 22 2002 Simon Matter <simon.matter@invoca.ch>
d0d307
# - added two-step conversion method
d0d307
#
d0d307
# v1.0.2, Jan 10 2003 Simon Matter <simon.matter@invoca.ch>
d0d307
# - fixed a bug where cvt_cyrusdb was called to convert empty or
d0d307
#   nonexistent files
d0d307
#
d0d307
# v1.0.3, Mar 14 2003 Simon Matter <simon.matter@invoca.ch>
d0d307
# - fixed a problem with new versions of the file command
d0d307
#
d0d307
# v1.0.4
d0d307
# - added GPL license
d0d307
#
d0d307
# v1.0.5, May 02 2003 Simon Matter <simon.matter@invoca.ch>
d0d307
# - modified exec path
d0d307
#
d0d307
# v1.0.6, Jul 18 2003 Simon Matter <simon.matter@invoca.ch>
d0d307
# - changed db3 to berkeley
d0d307
# - added new db backends for 2.2
d0d307
#
d0d307
# v1.0.7, Jan 23 2004 Simon Matter <simon.matter@invoca.ch>
d0d307
# - included some modifications from Luca Olivetti <luca@olivetti.cjb.net>
d0d307
# - added masssievec functionality
d0d307
#
d0d307
# v1.0.8, Jan 28 2004 Simon Matter <simon.matter@invoca.ch>
d0d307
# - convert sieve scripts to UTF-8 before calling masssievec
d0d307
#
d0d307
# v1.0.9, Jan 29 2004 Simon Matter <simon.matter@invoca.ch>
d0d307
# - convert sieve scripts to UTF-8 only if sievec failed before
d0d307
#
d0d307
# v1.0.10, Feb 24 2004 Simon Matter <simon.matter@invoca.ch>
d0d307
# - change su within init script to get input from
d0d307
#   /dev/null, this prevents hang when running in SELinux
d0d307
#
d0d307
# v1.0.11, Mar 02 2004 Simon Matter <simon.matter@invoca.ch>
d0d307
# - fixed SELinux fix
d0d307
#
d0d307
# v1.0.12, Dec 16 2004 Simon Matter <simon.matter@invoca.ch>
d0d307
# - use runuser instead of su if available
cf58a5
#
cf58a5
# v1.0.13, Jul 15 2005 Simon Matter <simon.matter@invoca.ch>
cf58a5
# - don't use flat in the two step conversion, use skiplist instead
cf58a5
#
cf58a5
# v1.0.14, Jul 18 2005 Simon Matter <simon.matter@invoca.ch>
cf58a5
# - replace the order of the magic files in the file call to make
cf58a5
#   sure skiplist is detected correctly.
cf58a5
#
cf58a5
# v1.0.15, Aug 17 2005 Simon Matter <simon.matter@invoca.ch>
cf58a5
# - add functionality to export all berkeley db files to skiplist
cf58a5
#
cf58a5
# v1.1.0, Aug 18 2005 Simon Matter <simon.matter@invoca.ch>
cf58a5
# - fix export functionality, try to recover Berkeley databases
cf58a5
#   as much as possible before any conversion.
cf58a5
#
cf58a5
# v1.1.1, Dec 05 2005 Simon Matter <simon.matter@invoca.ch>
cf58a5
# - run db_checkpoint in background with a timeout to prevent
cf58a5
#   that cyrus-imapd doesn't start at all if it hangs.
cf58a5
#
cf58a5
# v1.1.2, Dec 06 2005 Simon Matter <simon.matter@invoca.ch>
cf58a5
# - make handling of db_checkpoint more robust
cf58a5
#
cf58a5
# v1.2.0, Jan 12 2006 Simon Matter <simon.matter@invoca.ch>
cf58a5
# - adopt for cyrus-imapd-2.3
cf58a5
#
cf58a5
# v1.2.1, Jan 13 2006 Simon Matter <simon.matter@invoca.ch>
cf58a5
# - code cleanup
cf58a5
cf58a5
VERSION=1.2.1
d0d307
d0d307
if [ -n "$(/sbin/pidof cyrus-master)" ]; then
d0d307
  echo "ERROR: cyrus-master is running, unable to convert mailboxes!"
d0d307
  exit 1
d0d307
fi
d0d307
d0d307
if [ ! -f /etc/imapd.conf ]; then
d0d307
  echo "ERROR: configuration file not found."
d0d307
  exit 1
d0d307
fi
d0d307
d0d307
# fallback to su if runuser not available
d0d307
if [ -x /sbin/runuser ]; then
d0d307
  RUNUSER=runuser
d0d307
else
d0d307
  RUNUSER=su
d0d307
fi
d0d307
d0d307
# force cyrus user for security reasons
d0d307
if [ ! $(whoami) = "cyrus" ]; then
cf58a5
  exec $RUNUSER - cyrus -c "cd $PWD < /dev/null ; $0 $*"
d0d307
fi
d0d307
cf58a5
# special function for migration
cf58a5
EXPORT=$1
cf58a5
d0d307
# files get mode 0600
d0d307
umask 166
d0d307
cf58a5
# show version info in log files
cf58a5
echo "cvt_cyrusdb_all version: $VERSION"
cf58a5
d0d307
# get_config [config default]
d0d307
# extracts config option from config file
d0d307
get_config() {
d0d307
  if config=$(grep "^$1" /etc/imapd.conf); then
d0d307
    echo $config | cut -d: -f2 | sed -e 's/^ *//' -e 's/-nosync//' -e 's/ *$//'
d0d307
  else
d0d307
    echo $2
d0d307
  fi
d0d307
}
d0d307
d0d307
# where to find files and directories
cf58a5
data_dir=/usr/share/cyrus-imapd/rpm
cf58a5
lib_dir=/usr/lib/cyrus-imapd
d0d307
system_magic=$(file --version | awk '/magic file/ {print $4}')
cf58a5
cyrus_magic=${data_dir}/magic
cf58a5
cvt_cyrusdb=${lib_dir}/cvt_cyrusdb
cf58a5
sievec=${lib_dir}/sievec
cf58a5
masssievec=${lib_dir}/masssievec
d0d307
imap_prefix=$(get_config configdirectory /var/lib/imap)
d0d307
sieve_dir=$(get_config sievedir /var/lib/imap/sieve)
cf58a5
db_cfg=${data_dir}/db.cfg
d0d307
db_current=${imap_prefix}/rpm/db.cfg.current
d0d307
db_cache=${imap_prefix}/rpm/db.cfg.cache
d0d307
d0d307
# source default db backend config
d0d307
. $db_cfg
d0d307
d0d307
# get configured db backend config
cf58a5
duplicate_db=$(get_config duplicate_db $duplicate_db)
cf58a5
mboxlist_db=$(get_config mboxlist_db $mboxlist_db)
cf58a5
seenstate_db=$(get_config seenstate_db $seenstate_db)
cf58a5
subscription_db=$(get_config subscription_db $subscription_db)
cf58a5
tlscache_db=$(get_config tlscache_db $tlscache_db)
cf58a5
annotation_db=$(get_config annotation_db $annotation_db)
cf58a5
mboxkey_db=$(get_config mboxkey_db $mboxkey_db)
cf58a5
ptscache_db=$(get_config ptscache_db $ptscache_db)
cf58a5
quota_db=$(get_config quota_db $quota_db)
d0d307
d0d307
# remember current db backend config
d0d307
{
cf58a5
echo "duplicate_db=$duplicate_db"
cf58a5
echo "mboxlist_db=$mboxlist_db"
cf58a5
echo "seenstate_db=$seenstate_db"
cf58a5
echo "subscription_db=$subscription_db"
cf58a5
echo "tlscache_db=$tlscache_db"
cf58a5
echo "annotation_db=$annotation_db"
cf58a5
echo "mboxkey_db=$mboxkey_db"
cf58a5
echo "ptscache_db=$ptscache_db"
cf58a5
echo "quota_db=$quota_db"
cf58a5
echo "sieve_version=$sieve_version"
cf58a5
} | sort > $db_current
d0d307
d0d307
# file_type [file]
d0d307
file_type() {
cf58a5
  this_type=$(file -b -m "$cyrus_magic:$system_magic" "$1" 2> /dev/null)
d0d307
  if echo "$this_type" | grep -qi skip > /dev/null 2>&1; then
d0d307
    echo skiplist
d0d307
  elif echo "$this_type" | grep -qi text > /dev/null 2>&1; then
d0d307
    echo flat
d0d307
  else
d0d307
    echo berkeley
d0d307
  fi
d0d307
}
d0d307
d0d307
# cvt_file [file] [db]
d0d307
cvt_file() {
d0d307
  target="$1"
d0d307
  new_db="$2"
d0d307
  if [ -s "$target" ]; then
d0d307
    old_db=$(file_type "$target")
d0d307
    if [ ! "$old_db" = "$new_db" ]; then
d0d307
      # The two-step conversion is paranoia against the filenames being encoded
d0d307
      # inside the database or logfiles (berkeley does this, for example).
cf58a5
      rm -f "${target}.skiplist"
cf58a5
      if [ "$old_db" = "skiplist" ]; then
cf58a5
        cp -a "$target" "${target}.skiplist"
d0d307
      else
cf58a5
        $cvt_cyrusdb "$target" "$old_db" "${target}.skiplist" skiplist
d0d307
      fi
d0d307
      RETVAL=$?
d0d307
      ERRVAL=$[ $ERRVAL + $RETVAL ]
d0d307
      if [ $RETVAL -eq 0 ]; then
d0d307
        rm -f "$target"
cf58a5
        if [ -s "${target}.skiplist" ]; then
cf58a5
          if [ "$new_db" = "skiplist" ]; then
cf58a5
            cp -a "${target}.skiplist" "$target"
d0d307
          else
cf58a5
            $cvt_cyrusdb "${target}.skiplist" skiplist "$target" "$new_db"
d0d307
          fi
d0d307
        fi
d0d307
        RETVAL=$?
d0d307
        ERRVAL=$[ $ERRVAL + $RETVAL ]
d0d307
        if [ $RETVAL -eq 0 ]; then
cf58a5
          rm -f "${target}.skiplist"
d0d307
        else
cf58a5
          echo "ERROR: unable to convert ${target}.skiplist from skiplist to $new_db"
d0d307
        fi
d0d307
      else
cf58a5
        echo "ERROR: unable to convert $target from $old_db to skiplist"
d0d307
      fi
d0d307
    fi
d0d307
  fi
d0d307
}
d0d307
d0d307
# cvt_to_utf8 [file]
d0d307
cvt_to_utf8() {
d0d307
  target="$1"
d0d307
  if [ -s "$target" ]; then
d0d307
    if ! $sievec "$target" "${target}.sievec"; then
d0d307
      iconv --from-code=ISO-8859-1 --to-code=UTF-8 --output="${target}.UTF-8" "$target"
d0d307
      if [ -s "${target}.UTF-8" ]; then
d0d307
        # preserve timestamp
d0d307
        touch --reference="$target" "${target}.UTF-8"
d0d307
        mv -f "${target}.UTF-8" "$target"
d0d307
      else
d0d307
        ERRVAL=$[ $ERRVAL + 1 ]
d0d307
      fi
d0d307
    fi
d0d307
    rm -f "${target}.sievec"
d0d307
  fi
d0d307
}
d0d307
d0d307
ERRVAL=0
d0d307
cf58a5
# make sure our Berkeley databases are in a sane state
cf58a5
# wait for db_checkpoint to end successfully or kill it after a timeout
cf58a5
db_checkpoint -v -1 -h $imap_prefix/db &
cf58a5
DB_CHECK_PID=$!
cf58a5
CNT=0
cf58a5
while [ $CNT -lt 60 ]; do
cf58a5
  if ! kill -0 $DB_CHECK_PID > /dev/null 2>&1; then
cf58a5
    break
cf58a5
  fi
cf58a5
  sleep 1
cf58a5
  let CNT+=1
cf58a5
done
cf58a5
if kill -0 $DB_CHECK_PID > /dev/null 2>&1; then
cf58a5
  kill -USR1 $DB_CHECK_PID > /dev/null 2>&1
cf58a5
  sleep 1
cf58a5
  kill -KILL $DB_CHECK_PID > /dev/null 2>&1
cf58a5
  wait $DB_CHECK_PID > /dev/null 2>&1
cf58a5
fi
cf58a5
cf58a5
# do a normal recovery
cf58a5
db_recover -v -h $imap_prefix/db
cf58a5
RETVAL=$?
cf58a5
if [ $RETVAL -ne 0 ]; then
cf58a5
  # try a catastrophic recovery instead of normal recovery
cf58a5
  db_recover -v -c -h $imap_prefix/db
cf58a5
  RETVAL=$?
cf58a5
  ERRVAL=$[ $ERRVAL + $RETVAL ]
cf58a5
  if [ $RETVAL -ne 0 ]; then
cf58a5
    echo "ERROR: catastrophic recovery of Berkeley databases failed"
cf58a5
  fi
cf58a5
fi
cf58a5
cf58a5
if [ "$EXPORT" = "export" ]; then
cf58a5
  # convert all db files to skiplist for migration
cf58a5
  # TODO: quota_db, we don't touch it for now
cf58a5
  cvt_file $imap_prefix/deliver.db           "skiplist"
cf58a5
  cvt_file $imap_prefix/mailboxes.db         "skiplist"
cf58a5
  cvt_file $imap_prefix/tls_sessions.db      "skiplist"
cf58a5
  cvt_file $imap_prefix/annotations.db       "skiplist"
cf58a5
  cvt_file $imap_prefix/ptclient/ptscache.db "skiplist"
cf58a5
  rm -vf $imap_prefix/db/log.*
cf58a5
  rm -vf $imap_prefix/db/__db.*
cf58a5
else
cf58a5
  # always convert db files which have been converted to skiplist
cf58a5
  # TODO: quota_db, we don't touch it for now
cf58a5
  cvt_file $imap_prefix/deliver.db           "$duplicate_db"
cf58a5
  cvt_file $imap_prefix/mailboxes.db         "$mboxlist_db"
cf58a5
  cvt_file $imap_prefix/tls_sessions.db      "$tlscache_db"
cf58a5
  cvt_file $imap_prefix/annotations.db       "$annotation_db"
cf58a5
  cvt_file $imap_prefix/ptclient/ptscache.db "$ptscache_db"
cf58a5
  # do we have to convert all databases?
cf58a5
  if ! cmp -s $db_current $db_cache; then
cf58a5
    # we treat sieve scripts the same way like db files
cf58a5
    find ${sieve_dir}/ -name "*.script" -type f | while read db_file trash; do
cf58a5
      cvt_to_utf8 "$db_file"
cf58a5
    done
cf58a5
    $masssievec $sievec
cf58a5
    # convert all db files left
cf58a5
    find ${imap_prefix}/user/ -name "*.seen" -type f | while read db_file trash; do
cf58a5
      cvt_file "$db_file" "$seenstate_db"
cf58a5
    done
cf58a5
    find ${imap_prefix}/user/ -name "*.sub" -type f | while read db_file trash; do
cf58a5
      cvt_file "$db_file" "$subscription_db"
cf58a5
    done
cf58a5
    find ${imap_prefix}/user/ -name "*.mboxkey" -type f | while read db_file trash; do
cf58a5
      cvt_file "$db_file" "$mboxkey_db"
cf58a5
    done
cf58a5
  fi
d0d307
fi
d0d307
cf58a5
# update the config cache file so we can check whether something has changed
d0d307
if [ $ERRVAL -eq 0 ]; then
d0d307
  mv -f $db_current $db_cache
d0d307
else
d0d307
  rm -f $db_cache
d0d307
  rm -f $db_current
d0d307
fi
d0d307
d0d307
exit $ERRVAL