#!/usr/bin/perl
#============================================================= -*-perl-*-
#
# BackupPC_migrateV3toV4: Migrate a backup from V3 to V4 storage
#
# DESCRIPTION
#
#   BackupPC_migrateV3toV4: migrate a backup from V3 to V4 storage
#
#   Usage:
#       BackupPC_migrateV3toV4 [-p] [-v] [-h host [-n V3backupNum]]
#
# AUTHOR
#   Craig Barratt  <cbarratt@users.sourceforge.net>
#
# COPYRIGHT
#   Copyright (C) 2001-2017  Craig Barratt
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#========================================================================
#
# Version 4.1.3, released 3 Jun 2017.
#
# See http://backuppc.sourceforge.net.
#
#========================================================================

use strict;
no  utf8;
use lib "/usr/share/backuppc/lib";
use Getopt::Std;
use File::Copy;
use File::Path;
use Data::Dumper;
use Digest::MD5;
use Fcntl ':mode';

use BackupPC::Lib;
use BackupPC::XS qw( :all );
use BackupPC::DirOps;

my $ErrorCnt       = 0;
my $FileCnt        = 0;
my($FileCntNext, $FileCntTotal, $FileCntPctTextLast, $FileCntIncr);

die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );
my $TopDir = $bpc->TopDir();
my $BinDir = $bpc->BinDir();
my %Conf   = $bpc->Conf();
my $Hosts  = $bpc->HostInfoRead();

my %opts;

if ( !getopts("ampvh:n:", \%opts) || @ARGV >= 1
        || !(defined($opts{a}) xor defined($opts{h}))
        || (!defined($opts{h}) && defined($opts{n})) ) {
    print STDERR <<EOF;
usage: BackupPC_migrateV3toV4 -a [-m] [-p] [-v]
       BackupPC_migrateV3toV4 -h host [-n V3backupNum] [-m] [-p] [-v]
  Options:
     -a              migrate all hosts and all backups
     -h host         migrate just a specific host
     -n V3backupNum  migrate specific host backup; does all V3 backups
                     for that host if not specified
     -m              don't migrate anything; just print what would be done
     -p              don't print progress information
     -v              verbose
EOF
    exit(1);
}

#
# Make sure BackupPC isn't running
#
CheckIfServerRunning();

if ( defined($opts{h}) ) {
    if ( $opts{h} !~ /^([\w\.\s-]+)$/
            || $opts{h} =~ m{(^|/)\.\.(/|$)}
            || !defined($Hosts->{$opts{h}}) ) {
        print(STDERR "BackupPC_migrateV3toV4: bad host name '$opts{h}'\n");
        exit(1);
    }
    if ( $opts{n} !~ /^\d*/ ) {
        print(STDERR "BackupPC_migrateV3toV4: invalid backup number '$opts{n}'\n");
        exit(1);
    }
}
$bpc->ChildInit();

my($Host, $LogLevel, $Compress, $SrcDir, $DestDir, $BkupNum, $Inode, $DeltaInfo, $Inode2MD5Digest);

foreach my $h ( defined($opts{h}) ? ($opts{h}) : sort(keys(%$Hosts)) ) {
    $Host = $h;
    if ( defined(my $error = $bpc->ConfigRead($Host)) ) {
        print("BackupPC_migrateV3toV4: Can't read $Host's config file: $error\n");
        exit(1);
    }
    #
    # Make sure BackupPC isn't running
    #
    CheckIfServerRunning();

    %Conf = $bpc->Conf();
    $Conf{XferLogLevel} = 8 if ( $opts{v} );
    BackupPC::XS::Lib::logLevelSet($Conf{XferLogLevel});
    $LogLevel = $Conf{XferLogLevel};
    if ( $LogLevel < 4 ) {
        select(STDOUT); $| = 1;
    }

    #
    # Write new-style attrib files (<= 4.0.0beta3 uses old-style), which are 0-length
    # files with the digest encoded in the file name (eg: attrib_md5HexDigest). We
    # can still read the old-style files, and we upgrade them as we go.
    #
    BackupPC::XS::Attrib::backwardCompat(0, 0);

    my @Backups = $bpc->BackupInfoRead($Host);

    $Inode = 1;
    my $bkupNum2LastInode = {};
    my $foundBackup = 0;
    for ( my $i = 0 ; $i < @Backups ; $i++ ) {
        $Inode = $Backups[$i]{inodeLast} + 1 if ( $Inode < $Backups[$i]{inodeLast} );
        $foundBackup = 1 if ( $opts{n} == $Backups[$i]{num} );
    }
    $Inode2MD5Digest = {};
    $BkupNum = undef;
    if ( defined($opts{n}) && !$foundBackup ) {
        print("BackupPC_migrateV3toV4: backup $opts{n} for host $Host doesn't exist\n");
        next;
    }
    for ( my $i = 0 ; $i < @Backups ; $i++ ) {
        next if ( (defined($opts{n}) && $opts{n} != $Backups[$i]{num}) );
        if ( $Backups[$i]{version} >= 4 ) {
            if ( defined($opts{n}) ) {
                print("BackupPC_migrateV3toV4: backup $Backups[$i]{num} for host $Host is version >= 4; skipping\n");
            }
            next;
        }
        $BkupNum  = $Backups[$i]{num};
        $Compress = $Backups[$i]{compress};
        $SrcDir   = "$TopDir/pc/$Host/$BkupNum";
        $DestDir  = "$TopDir/pc/$Host/$BkupNum.v4";
        $FileCnt  = 0;
        $FileCntNext = 1;
        $FileCntTotal = $Backups[$i]{nFiles};
        $FileCntIncr  = int($FileCntTotal / 500);
        $FileCntIncr  = 2 if ( $FileCntIncr < 2 );
        #
        #
        # Migrate the V3 backup from $SrcDir to $DestDir
        #
        if ( -d "$SrcDir/refCnt" ) {
            print("BackupPC_migrateV3toV4: backup $Backups[$i]{num} for host $Host already has a refCnt directory; skipping this backup\n");
            next;
        }
        print("BackupPC_migrateV3toV4: migrating host $Host backup #$BkupNum to V4 (approx $FileCntTotal files)\n") if ( !$opts{p} );
        next if ( $opts{m} );

        if ( -d $DestDir ) {
            print("BackupPC_migrateV3toV4: removing temp target directory $DestDir\n");
            BackupPC::DirOps::RmTreeQuiet($bpc, $DestDir, -2);
            if ( -d $DestDir ) {
                print("BackupPC_migrateV3toV4: failed to remove temp target directory $DestDir; skipping this backup\n");
                next;
            }
        }

        createFsckFile($DestDir);
        $bpc->flushXSLibMesgs();

        $DeltaInfo = BackupPC::XS::DeltaRefCnt::new($DestDir);
	if ( !chdir($SrcDir) ) {
	    print("BackupPC_migrateV3toV4: cannot chdir to $SrcDir; skipping this backup\n");
	    next;
	}

        update_v3_dir("attrib", ".");
        BackupPC::DirOps::find($bpc, {wanted => \&update_v3_dir}, ".", 1);
        $DeltaInfo->flush();
        $bpc->flushXSLibMesgs();
        printProgress();
        if ( (my $errCnt = BackupPC::XS::Lib::logErrorCntGet()) > 0 ) {
            print("BackupPC_migrateV3toV4: got $errCnt errors from BackupPC::XS::Lib::logErrorCntGet()\n");
            $ErrorCnt += $errCnt;
        }
        $bpc->flushXSLibMesgs();
        unlink("$DestDir/refCnt/needFsck.mig") if ( $ErrorCnt == 0 );
        $bkupNum2LastInode->{$Backups[$i]{num}} = $Inode;
        #
        # move $DestDir to $SrcDir and then remove $SrcDir
        #
        if ( -e "$SrcDir.old" ) {
            if ( -d "$SrcDir.old" ) {
                BackupPC::DirOps::RmTreeQuiet($bpc, "$SrcDir.old", -2);
            } else {
                unlink("$SrcDir.old");
            }
        }
        if ( !rename($SrcDir, "$SrcDir.old") ) {
            print("BackupPC_migrateV3toV4: unable to rename $SrcDir to $SrcDir.old; cleaning up by removing $DestDir\n");
            BackupPC::DirOps::RmTreeQuiet($bpc, $DestDir, -2);
            $ErrorCnt++;
            next;
        }
        if ( !rename($DestDir, $SrcDir) ) {
            $ErrorCnt++;
            print("BackupPC_migrateV3toV4: unable to rename $DestDir to $SrcDir; moving $SrcDir.old back to $SrcDir and removing $DestDir\n");
            if ( !rename("$SrcDir.old", $SrcDir) ) {
                print("BackupPC_migrateV3toV4: unable to rename $SrcDir.old back to $SrcDir\n");
                $ErrorCnt++;
                next;
            }
            BackupPC::DirOps::RmTreeQuiet($bpc, $DestDir, -2);
        }
        print("BackupPC_migrateV3toV4: converted backup in $SrcDir; removing $SrcDir.old\n");
        BackupPC::DirOps::RmTreeQuiet($bpc, "$SrcDir.old", -2);
    }
    next if ( $opts{m} );
    if ( system("$BinDir/BackupPC_refCountUpdate -h $Host -o 0") != 0 ) {
        print("BackupPC_migrateV3toV4: BackupPC_refCountUpdate exited with errors\n");
        $ErrorCnt++
    }
    #
    # Update last inode in each backup we updated
    #
    @Backups = $bpc->BackupInfoRead($Host);
    my $updateBackups = 0;
    for ( my $i = 0 ; $i < @Backups ; $i++ ) {
        next if ( !defined($bkupNum2LastInode->{$Backups[$i]{num}}) );
        $Backups[$i]{inodeLast} = $bkupNum2LastInode->{$Backups[$i]{num}};
        print("Updating inodeLast for backup #$Backups[$i]{num} to $Backups[$i]{inodeLast}\n");
        BackupPC::Storage->backupInfoWrite("$TopDir/pc/$Host", $Backups[$i]{num}, $Backups[$i], 1);
        $updateBackups = 1;
    }
    $bpc->BackupInfoWrite($Host, @Backups) if ( $updateBackups );
}

print("BackupPC_migrateV3toV4: got $ErrorCnt error" . ($ErrorCnt == 1 ? "\n" : "s\n"));
exit($ErrorCnt ? 1 : 0);

sub update_v3_dir
{
    my($name, $path) = @_;

    print("update_v3_dir name = $name, path = $path\n") if ( $LogLevel >= 6 );
    return if ( !-d $path );

    $name = "attrib";

    my $attrOld = BackupPC::XS::Attrib::new($Compress);
    my $attr    = BackupPC::XS::Attrib::new($Compress);
    if ( -f "$path/attrib" && !$attrOld->read($path, $name) ) {
        print("BackupPC_migrateV3toV4: can't read attribute file $path/$name\n");
        $ErrorCnt++;
    }
    my $attrAll = {};
    #
    # populate attrAll with the files in the directory
    #
    my $dirEntries = BackupPC::DirOps::dirRead($bpc, $path);
    foreach my $e ( @$dirEntries ) {
	my $file = $e->{name};
	next if ( $file !~ /^f/ );
	my $fileUM = $bpc->fileNameUnmangle($file);
        my $a = $attrOld->get($fileUM);
        if ( defined($a) ) {
            $attrAll->{$fileUM} = $a;
            next;
        }
        #
        # We found a file that isn't in the attributes.  Need to create an
        # entry for it.
        #
	my @s = stat("$path/$file");
        my $size = 0;
        if ( S_ISREG($s[2]) ) {
            if ( !$Compress ) {
                $size = $s[7];
            } else {
                #
                # Compute the correct size by reading the whole file
                #
                my $f = BackupPC::XS::FileZIO::open("$path/$file", 0, $Compress);
                if ( !defined($f) ) {
                    print("BackupPC_migrateV3toV4: can't open $path/$file");
                    $ErrorCnt++;
                } else {
                    my($data);
                    while ( $f->read(\$data, 65636 * 8) > 0 ) {
                        $size += length($data);
                    }
                    $f->close;
                }
            }
        }
        $attrAll->{$fileUM} = {
            name  => $fileUM,
	    type  => S_ISDIR($s[2]) ? BPC_FTYPE_DIR : BPC_FTYPE_FILE,
	    mode  => $s[2],
	    uid   => $s[4],
	    gid   => $s[5],
	    size  => $size,
	    mtime => $s[9],
	};
        print("update_v3_dir: adding missing attributes for $path/$file (type $attrAll->{$fileUM}{type}, size $size, ...)\n")
                            if ( $LogLevel >= 4 );
    }

    foreach my $fileUM ( keys(%$attrAll) ) {
        my $a = $attrAll->{$fileUM};
	my $copyFile = 0;

        $FileCnt++ if ( $a->{type} != BPC_FTYPE_DIR );
        $a->{inode}  = $Inode++;
        $a->{nlinks} = 0;

	print("update_v3_dir($path): got file $a->{name}\n") if ( $LogLevel >= 6 );

	$copyFile = 1 if ( $a->{type} == BPC_FTYPE_FILE
			|| $a->{type} == BPC_FTYPE_SYMLINK
			|| $a->{type} == BPC_FTYPE_HARDLINK
			|| $a->{type} == BPC_FTYPE_CHARDEV
			|| $a->{type} == BPC_FTYPE_BLOCKDEV );

        #
        # If there isn't an associated file, or there is a digest already, then
        # there nothing more to do
        #
	if ( !$copyFile || length($a->{digest}) ) {
            $attr->set($a->{name}, $a);
            next;
        }

        #
        # Check if we've seen this file already based on the fullPath inode
        #
        my $fullPath = "$path/" . $bpc->fileNameEltMangle($a->{name});
        my @s = stat($fullPath);
        if ( @s == 0 ) {
            print("BackupPC_migrateV3toV4: unable to stat $fullPath\n");
            $ErrorCnt++;
            next;
        }
	my $inode = $s[1];
        if ( defined($Inode2MD5Digest->{$inode}) ) {
            $a->{digest} = $Inode2MD5Digest->{$inode};
            if ( $LogLevel >= 5 ) {
                my $digestStr = unpack("H*", $a->{digest});
                print("$a->{name}: found inode $inode -> digest = $digestStr\n") if ( $LogLevel >= 4 );
            }
            $attr->set($a->{name}, $a);
            next;
        }

        #
        # Compute the V3 and V4 digests
        #
	my $f = BackupPC::XS::FileZIO::open($fullPath, 0, $Compress);
	my($size, $data, $data1MB, $fileSize, $found, @s, $thisRead);
	if ( !defined($f) ) {
	    print("BackupPC_migrateV3toV4: unable to open file $fullPath\n");
	    $ErrorCnt++;
            next;
	}
	my $md5 = Digest::MD5->new();
	do {
	    $thisRead = $f->read(\$data, 1 << 20);
	    if ( $thisRead < 0 ) {
		print("BackupPC_migrateV3toV4: error reading file $fullPath (read returns $thisRead)\n");
		$ErrorCnt++;
		$f->close();
		next;
	    } elsif ( $thisRead > 0 ) {
		$md5->add($data);
		$fileSize += length($data);
	    }
	} while ( $thisRead > 0 );
	my $digest4 = $md5->digest();
	my $digest3 = $bpc->Buffer2MD5_v3(Digest::MD5->new(), $fileSize, \$data1MB);
	my $path3 = $bpc->MD52Path_v3($digest3, $Compress);
	my $path4 = $bpc->MD52Path($digest4, $Compress);
	my $i = -1;

	print("$a->{name}: path3 = $path3, path4 = $path4\n") if ( $LogLevel >= 5 );

	#
	# see if it's already in the v4 pool
	#
	if ( $fileSize == 0 ) {
	    $found = 1;
	    print("$a->{name}: empty file in pool by default\n") if ( $LogLevel >= 4 );
	} elsif ( (@s = stat($path4)) && $s[1] == $inode ) {
	    $found = 1;
	    print("$a->{name}: already in pool4 $path4\n") if ( $LogLevel >= 4 );
	} elsif ( !-f $path4 ) {
	    #
	    # look in v3 pool for match
	    #
	    while ( 1 ) {
		my $testPath = $path3;
		$testPath .= "_$i" if ( $i >= 0 );
		@s = stat($testPath);
		last if ( !@s );
		if ( $s[1] != $inode ) {
		    $i++;
		    next;
		}
		#
		# Found it!
		#
		print("$a->{name}: found at $testPath; link/move to $path4\n") if ( $LogLevel >= 4 );
		$found = 1;
		my $dir4 = $path4;
		$dir4 =~ s{(.*)/.*}{$1};
		if ( !-d $dir4 ) {
		    print("Creating directory $dir4\n") if ( $LogLevel >= 5 );
		    eval { mkpath($dir4, 0, 0777) };
		    if ( $@ ) {
			print("BackupPC_migrateV3toV4: can't mkpath $dir4\n");
			$ErrorCnt++;
			$found = 0;
		    }
		}
		if ( !link($testPath, $path4) ) {
		    print("BackupPC_migrateV3toV4: can't link($testPath,$path4)\n");
		    $ErrorCnt++;
		    $found = 0;
		} elsif ( unlink($testPath) != 1 ) {
		    print("BackupPC_migrateV3toV4: can't unlink($testPath)\n");
		    $ErrorCnt++;
		    $found = 0;
		}
		last;
	    }
	}
	#
	# check one more time for the V4 pool in case someone else just moved it there
	# (in which case the link error above is actually not an error).
	#
	if ( !$found && (@s = stat($path4)) && $s[1] == $inode ) {
	    $found = 1;
	    print("$a->{name}: rediscovered in pool4 $path4; prior link error is benign\n");
	    $ErrorCnt-- if ( $ErrorCnt > 0 );        # reverse increment above
	}

	if ( $found ) {
	    $f->close();
	} else {
	    #
	    # fall back on copying it to the new pool
	    #
	    $f->rewind();
	    my $pw = BackupPC::XS::PoolWrite::new($Compress);
	    while ( $f->read(\$data, 1 << 20) > 0 ) {
		$pw->write(\$data);
		$size += length($data);
	    }
	    $f->close();
	    my($match, $poolSize, $errorCnt);
	    ($match, $digest4, $poolSize, $errorCnt) = $pw->close();
	    if ( $LogLevel >= 5 ) {
		my $digestStr = unpack("H*", $digest4);
		print("poolWrite->close $a->{name} returned match $match, digest $digestStr, poolSize $poolSize, errCnt $errorCnt\n")
                                            if ( $LogLevel >= 6 );
	    }
	    $ErrorCnt += $errorCnt;
	}
        $a->{digest} = $digest4;
        $attr->set($a->{name}, $a);
        $Inode2MD5Digest->{$inode} = $digest4;
        $DeltaInfo->update($Compress, $digest4, 1) if ( length($digest4) );
    }
    eval { mkpath("$DestDir/$path", 0, 0777) };
    if ( $@ ) {
        print("BackupPC_migrateV3toV4: can't mkpath $DestDir/$path\n");
        $ErrorCnt++;
    }
    if ( !$attr->write("$DestDir/$path", $name, undef, $DeltaInfo) ) {
	print("BackupPC_migrateV3toV4: write to $DestDir/$path/$name failed\n");
	$ErrorCnt++;
    }
    printProgress() if ( $FileCnt >= $FileCntNext );
    $bpc->flushXSLibMesgs();
}

#
# Create a needFsck file, so if we are killed and can't recover, we can
# make sure an fsck is run next time.
#
sub createFsckFile
{
    my($destDir) = @_;
    mkpath("$destDir/refCnt", 0, 0777);
    my $fh;
    if ( !(open($fh, ">", "$destDir/refCnt/needFsck.mig") && close($fh)) ) {
        print("BackupPC_migrateV3toV4: can't create $destDir/refCnt/needFsck.mig ($?)\n");
        $ErrorCnt++;
    }
    if ( !(open($fh, ">", "$destDir/refCnt/noPoolCntOk") && close($fh)) ) {
        print("BackupPC_migrateV3toV4: can't create $destDir/refCnt/noPoolCntOk ($?)\n");
        $ErrorCnt++;
    }
}

sub printProgress
{
    $FileCntNext = $FileCnt + $FileCntIncr;
    return if ( $opts{p} );
    my $pctText = sprintf("%.1f", 100 * $FileCnt / ($FileCntTotal > 0 ? $FileCntTotal : 1));
    return if ( $pctText eq $FileCntPctTextLast );
    print("$Host #$BkupNum: $pctText%\n");
    $FileCntPctTextLast = $pctText;
}

sub CheckIfServerRunning
{
    return if ( $opts{m} );
    my $err = $bpc->ServerConnect($Conf{ServerHost}, $Conf{ServerPort});
    if ( $err eq "" ) {
        print STDERR "$0: can't run since BackupPC is running\n";
        exit(1);
    }
}
