#!/usr/bin/env python
#
# Public Domain 2014-2018 MongoDB, Inc.
# Public Domain 2008-2014 WiredTiger, Inc.
#
# This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

import Queue
import threading, time, wiredtiger, wttest
import glob, os, shutil
from helper import compare_files
from suite_subprocess import suite_subprocess
from wtdataset import SimpleDataSet, simple_key
from wtscenario import make_scenarios
from wtthread import op_thread

# test_backup04.py
#    Utilities: wt backup
# Test cursor backup with target URIs
class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess):
    dir='backup.dir'                    # Backup directory name
    logmax="100K"

    # This test is written to test incremental backups.  We want to add
    # enough data to generate more than one log file each time we add data.
    # First we populate and take a full backup.  Then we loop, checkpointing
    # running an incremental backup with a targeted cursor and then calling
    # truncate to archive the logs.
    #
    # At the same time, we take a full backup during each loop.
    # We run recovery and verify the full backup with the incremental
    # backup after each loop.  We compare two backups instead of the original
    # because running 'wt' causes all of our original handles to be closed
    # and that is not what we want here.
    #
    pfx = 'test_backup'
    scenarios = make_scenarios([
        ('table', dict(uri='table:test',dsize=100,nops=2000,nthreads=1,time=30)),
    ])

    # Create a large cache, otherwise this test runs quite slowly.
    def conn_config(self):
        return 'cache_size=1G,log=(archive=false,enabled,file_max=%s)' % \
            self.logmax

    def populate(self, uri, dsize, rows):
        self.pr('populate: ' + uri + ' with ' + str(rows) + ' rows')
        cursor = self.session.open_cursor(uri, None)
        for i in range(1, rows + 1):
            cursor[simple_key(cursor, i)] = str(i) + ':' + 'a' * dsize
        cursor.close()

    def update(self, uri, dsize, upd, rows):
        self.pr('update: ' + uri + ' with ' + str(rows) + ' rows')
        cursor = self.session.open_cursor(uri, None)
        for i in range(1, rows + 1):
            cursor[simple_key(cursor, i)] = str(i) + ':' + upd * dsize
        cursor.close()

    # Compare the original and backed-up files using the wt dump command.
    def compare(self, uri, dir_full, dir_incr):
        # print "Compare: full URI: " + uri + " with incremental URI "
        if dir_full == None:
            full_name='original'
        else:
            full_name='backup_full'
        incr_name='backup_incr'
        if os.path.exists(full_name):
            os.remove(full_name)
        if os.path.exists(incr_name):
            os.remove(incr_name)
        #
        # We have been copying the logs only, so we need to force 'wt' to
        # run recovery in order to apply all the logs and check the data.
        #
        if dir_full == None:
            self.runWt(['-R', 'dump', uri], outfilename=full_name)
        else:
            self.runWt(['-R', '-h', dir_full, 'dump', uri], outfilename=full_name)
        self.runWt(['-R', '-h', dir_incr, 'dump', uri], outfilename=incr_name)
        self.assertEqual(True,
            compare_files(self, full_name, incr_name))

    def take_full_backup(self, dir):
        # Open up the backup cursor, and copy the files.  Do a full backup.
        cursor = self.session.open_cursor('backup:', None, None)
        self.pr('Full backup to ' + dir + ': ')
        os.mkdir(dir)
        while True:
            ret = cursor.next()
            if ret != 0:
                break
            newfile = cursor.get_key()
            sz = os.path.getsize(newfile)
            self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + dir)
            shutil.copy(newfile, dir)
        self.assertEqual(ret, wiredtiger.WT_NOTFOUND)
        cursor.close()

    # Take an incremental backup and then truncate/archive the logs.
    def take_incr_backup(self, dir):
            config = 'target=("log:")'
            cursor = self.session.open_cursor('backup:', None, config)
            while True:
                ret = cursor.next()
                if ret != 0:
                    break
                newfile = cursor.get_key()
                sz = os.path.getsize(newfile)
                self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + dir)
                shutil.copy(newfile, dir)
            self.assertEqual(ret, wiredtiger.WT_NOTFOUND)
            self.session.truncate('log:', cursor, None, None)
            cursor.close()

    # Run background inserts while running checkpoints and incremental backups
    # repeatedly.
    def test_incremental_backup(self):
        import sys
        # Create the backup directory.
        self.session.create(self.uri, "key_format=S,value_format=S")

        self.populate(self.uri, self.dsize, self.nops)

        # We need to start the directory for the incremental backup with
        # a full backup.  The full backup function creates the directory.
        dir = self.dir
        self.take_full_backup(dir)
        self.session.checkpoint(None)

        #
        # Incremental backups perform a loop:
        #   Do more work
        #   Checkpoint
        #   Copy log files returned by log targeted backup cursor
        #   Truncate (archive) the log files
        #   Close the backup cursor
        updstr="bcdefghi"
        for increment in range(0, 5):
            full_dir = self.dir + str(increment)
            # Add more work to move the logs forward.
            self.update(self.uri, self.dsize, updstr[increment], self.nops)
            self.session.checkpoint(None)

            self.pr('Iteration: ' + str(increment))
            self.take_incr_backup(self.dir)

        # After running, take a full backup.  Compare the incremental
        # backup to the original database and the full backup database.
        self.take_full_backup(full_dir)
        self.compare(self.uri, full_dir, self.dir)
        self.compare(self.uri, None, self.dir)

if __name__ == '__main__':
    wttest.run()
