/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.store;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat;
import org.apache.lucene.index.*;
import org.apache.lucene.store.*;
import org.apache.lucene.util.*;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.ExceptionsHelper;
import org.apache.lucene.util.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.Compressor;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.Callback;
import org.elasticsearch.common.util.SingleObjectCache;
import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
import org.elasticsearch.common.util.concurrent.RefCounted;
import org.elasticsearch.env.ShardLock;
import org.elasticsearch.index.CloseableIndexComponent;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.AbstractIndexShardComponent;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.distributor.Distributor;

import java.io.*;
import java.nio.file.NoSuchFileException;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.zip.Adler32;
import java.util.zip.CRC32;
import java.util.zip.Checksum;

/**
 * A Store provides plain access to files written by an elasticsearch index shard. Each shard
 * has a dedicated store that is uses to access Lucene's Directory which represents the lowest level
 * of file abstraction in Lucene used to read and write Lucene indices.
 * This class also provides access to metadata information like checksums for committed files. A committed
 * file is a file that belongs to a segment written by a Lucene commit. Files that have not been committed
 * ie. created during a merge or a shard refresh / NRT reopen are not considered in the MetadataSnapshot.
 * <p/>
 * Note: If you use a store it's reference count should be increased before using it by calling #incRef and a
 * corresponding #decRef must be called in a try/finally block to release the store again ie.:
 * <pre>
 *      store.incRef();
 *      try {
 *        // use the store...
 *
 *      } finally {
 *          store.decRef();
 *      }
 * </pre>
 */
public class Store extends AbstractIndexShardComponent implements CloseableIndexComponent, Closeable, RefCounted {

    private static final String CODEC = "store";
    private static final int VERSION_STACK_TRACE = 1; // we write the stack trace too since 1.4.0
    private static final int VERSION_START = 0;
    private static final int VERSION = VERSION_STACK_TRACE;
    private static final String CORRUPTED = "corrupted_";
    public static final String INDEX_STORE_STATS_REFRESH_INTERVAL = "index.store.stats_refresh_interval";

    private final AtomicBoolean isClosed = new AtomicBoolean(false);
    private final StoreDirectory directory;
    private final DistributorDirectory distributorDirectory;
    private final ReentrantReadWriteLock metadataLock = new ReentrantReadWriteLock();
    private final ShardLock shardLock;
    private final DirectoryService directoryService;
    private final OnClose onClose;
    private final SingleObjectCache<StoreStats> statsCache;

    private final AbstractRefCounted refCounter = new AbstractRefCounted("store") {
        @Override
        protected void closeInternal() {
            // close us once we are done
            Store.this.closeInternal();
        }
    };

    public Store(ShardId shardId, @IndexSettings Settings indexSettings, DirectoryService directoryService, Distributor distributor, ShardLock shardLock) throws IOException {
        this(shardId, indexSettings, directoryService, distributor, shardLock, OnClose.EMPTY);
    }

    @Inject
    public Store(ShardId shardId, @IndexSettings Settings indexSettings, DirectoryService directoryService, Distributor distributor, ShardLock shardLock, OnClose onClose) throws IOException {
        super(shardId, indexSettings);
        this.directoryService = directoryService;
        this.directory = new StoreDirectory(directoryService.newFromDistributor(distributor));

        distributorDirectory = DirectoryUtils.getLeaf(directory, DistributorDirectory.class);
        this.shardLock = shardLock;
        this.onClose = onClose;
        final TimeValue refreshInterval = indexSettings.getAsTime(INDEX_STORE_STATS_REFRESH_INTERVAL, TimeValue.timeValueSeconds(10));

        this.statsCache = new StoreStatsCache(refreshInterval, directory, directoryService);
        logger.debug("store stats are refreshed with refresh_interval [{}]", refreshInterval);

        assert onClose != null;
        assert shardLock != null;
        assert shardLock.getShardId().equals(shardId);
    }

    public Directory directory() {
        ensureOpen();
        return directory;
    }

    /**
     * Returns the last committed segments info for this store
     *
     * @throws IOException if the index is corrupted or the segments file is not present
     */
    public SegmentInfos readLastCommittedSegmentsInfo() throws IOException {
        failIfCorrupted();
        try {
            return readSegmentsInfo(null, directory());
        } catch (CorruptIndexException ex) {
            markStoreCorrupted(ex);
            throw ex;
        }
    }

    /**
     * Returns the segments info for the given commit or for the latest commit if the given commit is <code>null</code>
     *
     * @throws IOException if the index is corrupted or the segments file is not present
     */
    private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory directory) throws IOException {
        try {
            return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit, directory);
        } catch (EOFException eof) {
            // TODO this should be caught by lucene - EOF is almost certainly an index corruption
            throw new CorruptIndexException("Read past EOF while reading segment infos", eof);
        } catch (IOException exception) {
            throw exception; // IOExceptions like too many open files are not necessarily a corruption - just bubble it up
        } catch (Exception ex) {
            throw new CorruptIndexException("Hit unexpected exception while reading segment infos", ex);
        }

    }

    final void ensureOpen() {
        if (this.refCounter.refCount() <= 0) {
            throw new AlreadyClosedException("store is already closed");
        }
    }

    /**
     * Returns a new MetadataSnapshot for the latest commit in this store or
     * an empty snapshot if no index exists or can not be opened.
     *
     * @throws CorruptIndexException if the lucene index is corrupted. This can be caused by a checksum mismatch or an
     *                               unexpected exception when opening the index reading the segments file.
     */
    public MetadataSnapshot getMetadataOrEmpty() throws IOException {
        try {
            return getMetadata(null);
        } catch (IndexNotFoundException ex) {
            // that's fine - happens all the time no need to log
        } catch (FileNotFoundException | NoSuchFileException ex) {
            logger.info("Failed to open / find files while reading metadata snapshot");
        }
        return MetadataSnapshot.EMPTY;
    }

    /**
     * Returns a new MetadataSnapshot for the latest commit in this store.
     *
     * @throws CorruptIndexException  if the lucene index is corrupted. This can be caused by a checksum mismatch or an
     *                                unexpected exception when opening the index reading the segments file.
     * @throws FileNotFoundException  if one or more files referenced by a commit are not present.
     * @throws NoSuchFileException    if one or more files referenced by a commit are not present.
     * @throws IndexNotFoundException if no index / valid commit-point can be found in this store
     */
    public MetadataSnapshot getMetadata() throws IOException {
        return getMetadata(null);
    }

    /**
     * Returns a new MetadataSnapshot for the given commit. If the given commit is <code>null</code>
     * the latest commit point is used.
     *
     * @throws CorruptIndexException  if the lucene index is corrupted. This can be caused by a checksum mismatch or an
     *                                unexpected exception when opening the index reading the segments file.
     * @throws FileNotFoundException  if one or more files referenced by a commit are not present.
     * @throws NoSuchFileException    if one or more files referenced by a commit are not present.
     * @throws IndexNotFoundException if the commit point can't be found in this store
     */
    public MetadataSnapshot getMetadata(IndexCommit commit) throws IOException {
        ensureOpen();
        failIfCorrupted();
        metadataLock.readLock().lock();
        try {
            return new MetadataSnapshot(commit, directory, logger);
        } catch (CorruptIndexException ex) {
            markStoreCorrupted(ex);
            throw ex;
        } finally {
            metadataLock.readLock().unlock();
        }
    }


    /**
     * Renames all the given files form the key of the map to the
     * value of the map. All successfully renamed files are removed from the map in-place.
     */
    public void renameTempFilesSafe(Map<String, String> tempFileMap) throws IOException {
        // this works just like a lucene commit - we rename all temp files and once we successfully
        // renamed all the segments we rename the commit to ensure we don't leave half baked commits behind.
        final Map.Entry<String, String>[] entries = tempFileMap.entrySet().toArray(new Map.Entry[tempFileMap.size()]);
        ArrayUtil.timSort(entries, new Comparator<Map.Entry<String, String>>() {
            @Override
            public int compare(Map.Entry<String, String> o1, Map.Entry<String, String> o2) {
                String left = o1.getValue();
                String right = o2.getValue();
                if (left.startsWith(IndexFileNames.SEGMENTS) || right.startsWith(IndexFileNames.SEGMENTS)) {
                    if (left.startsWith(IndexFileNames.SEGMENTS) == false) {
                        return -1;
                    } else if (right.startsWith(IndexFileNames.SEGMENTS) == false) {
                        return 1;
                    }
                }
                return left.compareTo(right);
            }
        });
        metadataLock.writeLock().lock();
        // we make sure that nobody fetches the metadata while we do this rename operation here to ensure we don't
        // get exceptions if files are still open.
        try (Lock writeLock = Lucene.acquireWriteLock(directory())) {
            for (Map.Entry<String, String> entry : entries) {
                String tempFile = entry.getKey();
                String origFile = entry.getValue();
                // first, go and delete the existing ones
                try {
                    directory.deleteFile(origFile);
                } catch (FileNotFoundException | NoSuchFileException e) {
                } catch (Throwable ex) {
                    logger.trace("failed to delete file [{}] (it may be missing, which is OK)", ex, origFile);
                }
                // now, rename the files... and fail it it won't work
                this.renameFile(tempFile, origFile);
                final String remove = tempFileMap.remove(tempFile);
                assert remove != null;
            }
        } finally {
            metadataLock.writeLock().unlock();
        }

    }

    public StoreStats stats() throws IOException {
        ensureOpen();
        return statsCache.getOrRefresh();
    }

    public void renameFile(String from, String to) throws IOException {
        ensureOpen();
        // distributorDirectory could be null if not using a
        // DirectoryDistributor, so use the directory service and the regular
        // directory to rename the file if so
        if (distributorDirectory != null) {
            distributorDirectory.renameFile(directoryService, from, to);
        } else {
            directoryService.renameFile(directory, from, to);
        }
    }

    /**
     * Increments the refCount of this Store instance.  RefCounts are used to determine when a
     * Store can be closed safely, i.e. as soon as there are no more references. Be sure to always call a
     * corresponding {@link #decRef}, in a finally clause; otherwise the store may never be closed.  Note that
     * {@link #close} simply calls decRef(), which means that the Store will not really be closed until {@link
     * #decRef} has been called for all outstanding references.
     * <p/>
     * Note: Close can safely be called multiple times.
     *
     * @throws AlreadyClosedException iff the reference counter can not be incremented.
     * @see #decRef
     * @see #tryIncRef()
     */
    @Override
    public final void incRef() {
        refCounter.incRef();
    }

    /**
     * Tries to increment the refCount of this Store instance. This method will return <tt>true</tt> iff the refCount was
     * incremented successfully otherwise <tt>false</tt>. RefCounts are used to determine when a
     * Store can be closed safely, i.e. as soon as there are no more references. Be sure to always call a
     * corresponding {@link #decRef}, in a finally clause; otherwise the store may never be closed.  Note that
     * {@link #close} simply calls decRef(), which means that the Store will not really be closed until {@link
     * #decRef} has been called for all outstanding references.
     * <p/>
     * Note: Close can safely be called multiple times.
     *
     * @see #decRef()
     * @see #incRef()
     */
    @Override
    public final boolean tryIncRef() {
        return refCounter.tryIncRef();
    }

    /**
     * Decreases the refCount of this Store instance.If the refCount drops to 0, then this
     * store is closed.
     *
     * @see #incRef
     */
    @Override
    public final void decRef() {
        refCounter.decRef();
    }

    @Override
    public void close() {

        if (isClosed.compareAndSet(false, true)) {
            // only do this once!
            decRef();
        }
    }

    private void closeInternal() {
        try {
            try {
                directory.innerClose(); // this closes the distributorDirectory as well
            } finally {
                onClose.handle(shardLock);
            }
        } catch (IOException e) {
            logger.debug("failed to close directory", e);
        } finally {
            IOUtils.closeWhileHandlingException(shardLock);
        }
    }


    /**
     * Reads a MetadataSnapshot from the given index locations or returns an empty snapshot if it can't be read.
     *
     * @throws IOException if the index we try to read is corrupted
     */
    public static MetadataSnapshot readMetadataSnapshot(File[] indexLocations, ESLogger logger) throws IOException {
        final Directory[] dirs = new Directory[indexLocations.length];
        try {
            for (int i = 0; i < indexLocations.length; i++) {
                dirs[i] = new SimpleFSDirectory(indexLocations[i]);
            }
            DistributorDirectory dir = new DistributorDirectory(dirs);
            failIfCorrupted(dir, new ShardId("", 1));
            return new MetadataSnapshot(null, dir, logger);
        } catch (IndexNotFoundException ex) {
            // that's fine - happens all the time no need to log
        } catch (FileNotFoundException | NoSuchFileException ex) {
            logger.info("Failed to open / find files while reading metadata snapshot");
        } finally {
            IOUtils.close(dirs);
        }
        return MetadataSnapshot.EMPTY;
    }
    
    /** 
     * Returns true if the legacy checksum cannot be relied upon, because the file was not
     * append only. This only impacts Lucene 3.x files for elasticsearch.
     */
    static boolean isUnreliableLegacyChecksum(StoreFileMetaData metadata) {
        if (metadata.hasLegacyChecksum()) {
            // Lucene's .tii and .tis files (3.x) were not actually append-only.
            // this means the adler32 in ES 0.20.x releases is actually wrong, we can't use it.
            boolean badTermInfo = metadata.name().endsWith(".tii") || metadata.name().endsWith(".tis");
            // Lucene's .cfs (3.0-3.3) was not append-only, so old ES checksums (pre-0.18.x) are wrong.
            // NOTE: if we don't know the version, then we don't trust the checksum. 
            boolean badCFS = metadata.name().endsWith(".cfs") && 
                    (metadata.writtenBy() == null || metadata.writtenBy().onOrAfter(Version.LUCENE_34) == false);
            // Lucene's segments_N always had a checksum, and reasonably old versions of ES never added
            // their own metadata for it. But it wasn't append-only, so be defensive and don't rely upon
            // old versions omitting the checksum. NOTE: This logic also excludes segments.gen for the same reason.
            boolean badCommit = metadata.name().startsWith(IndexFileNames.SEGMENTS);
            
            return badTermInfo || badCFS || badCommit;
        }
        return false;
    }

    /**
     * Returns <code>true</code> iff the given location contains an index an the index
     * can be successfully opened. This includes reading the segment infos and possible
     * corruption markers.
     */
    public static boolean canOpenIndex(ESLogger logger, File... indexLocations) throws IOException {
        final Directory[] dirs = new Directory[indexLocations.length];
        try {
            for (int i = 0; i < indexLocations.length; i++) {
                dirs[i] = new SimpleFSDirectory(indexLocations[i]);
            }
            final Directory dir = dirs.length == 1 ? dirs[0] : new DistributorDirectory(dirs);
            failIfCorrupted(dir, new ShardId("", 1));
            Lucene.readSegmentInfos(dir);
            return true;
        } catch (Exception ex) {
            logger.trace("Can't open index for path [{}]", ex, indexLocations);
            return false;
        } finally {
            IOUtils.close(dirs);
        }

    }

    /**
     * The returned IndexOutput might validate the files checksum if the file has been written with a newer lucene version
     * and the metadata holds the necessary information to detect that it was been written by Lucene 4.8 or newer. If it has only
     * a legacy checksum, returned IndexOutput will not verify the checksum.
     * <p/>
     * Note: Checksums are calculated nevertheless since lucene does it by default since version 4.8.0. This method only adds the
     * verification against the checksum in the given metadata and does not add any significant overhead.
     */
    public IndexOutput createVerifyingOutput(String fileName, final StoreFileMetaData metadata, final IOContext context) throws IOException {
        IndexOutput output = directory().createOutput(fileName, context);
        boolean success = false;
        try {
            if (metadata.hasLegacyChecksum()) {
                if (isUnreliableLegacyChecksum(metadata)) {
                    logger.debug("create legacy length-only output for non-write-once file {}", fileName);
                    output = new LegacyVerification.LengthVerifyingIndexOutput(output, metadata.name(), metadata.length());
                } else {
                    logger.debug("create legacy adler32 output for {}", fileName);
                    output = new LegacyVerification.Adler32VerifyingIndexOutput(output, metadata.name(), metadata.checksum(), metadata.length());
                }
            } else if (metadata.checksum() == null) {
                // TODO: when the file is a segments_N, we can still CRC-32 + length for more safety
                // its had that checksum forever.
                logger.debug("create legacy length-only output for {}", fileName);
                output = new LegacyVerification.LengthVerifyingIndexOutput(output, metadata.name(), metadata.length());
            } else {
                assert metadata.writtenBy() != null;
                assert metadata.writtenBy().onOrAfter(Version.LUCENE_48);
                output = new LuceneVerifyingIndexOutput(metadata, output);
            }
            success = true;
        } finally {
            if (success == false) {
                IOUtils.closeWhileHandlingException(output);
            }
        }
        return output;

    }

    public static void verify(IndexOutput output) throws IOException {
        if (output instanceof VerifyingIndexOutput) {
            ((VerifyingIndexOutput) output).verify();
        }
    }

    public IndexInput openVerifyingInput(String filename, IOContext context, StoreFileMetaData metadata) throws IOException {
        if (metadata.hasLegacyChecksum() || metadata.checksum() == null) {
            logger.debug("open legacy input for {}", filename);
            return directory().openInput(filename, context);
        }
        assert metadata.writtenBy() != null;
        assert metadata.writtenBy().onOrAfter(Version.LUCENE_48);
        return new VerifyingIndexInput(directory().openInput(filename, context));
    }

    public static void verify(IndexInput input) throws IOException {
        if (input instanceof VerifyingIndexInput) {
            ((VerifyingIndexInput) input).verify();
        }
    }

    public boolean checkIntegrity(StoreFileMetaData md) {
        return checkIntegrity(md, directory());
    }

    public static boolean checkIntegrity(final StoreFileMetaData md, final Directory directory) {
        try (IndexInput input = directory.openInput(md.name(), IOContext.READONCE)) {
            if (input.length() != md.length()) { // first check the length no matter how old this file is
                return false;
            }
            if (md.writtenBy() != null && md.writtenBy().onOrAfter(Version.LUCENE_48)) {
                return Store.digestToString(CodecUtil.checksumEntireFile(input)).equals(md.checksum());
            } else if (md.hasLegacyChecksum() && !isUnreliableLegacyChecksum(md)) {
                // legacy checksum verification - no footer that we need to omit in the checksum!
                final Checksum checksum = new Adler32();
                final byte[] buffer = new byte[md.length() > 4096 ? 4096 : (int) md.length()];
                final long len = input.length();
                long read = 0;
                while (len > read) {
                    final long bytesLeft = len - read;
                    final int bytesToRead = bytesLeft < buffer.length ? (int) bytesLeft : buffer.length;
                    input.readBytes(buffer, 0, bytesToRead, false);
                    checksum.update(buffer, 0, bytesToRead);
                    read += bytesToRead;
                }
                return Store.digestToString(checksum.getValue()).equals(md.checksum());
            }
        } catch (IOException ex) {
            return false;
        }
        return true;
    }

    public boolean isMarkedCorrupted() throws IOException {
        ensureOpen();
        /* marking a store as corrupted is basically adding a _corrupted to all
         * the files. This prevent
         */
        final String[] files = directory().listAll();
        for (String file : files) {
            if (file.startsWith(CORRUPTED)) {
                return true;
            }
        }
        return false;
    }

    /**
     * Deletes all corruption markers from this store.
     */
    public void removeCorruptionMarker() throws IOException {
        ensureOpen();
        final Directory directory = directory();
        IOException firstException = null;
        final String[] files = directory.listAll();
        for (String file : files) {
            if (file.startsWith(CORRUPTED)) {
                try {
                    directory.deleteFile(file);
                } catch (IOException ex) {
                    if (firstException == null) {
                        firstException = ex;
                    } else {
                        firstException.addSuppressed(ex);
                    }
                }
            }
        }
        if (firstException != null) {
            throw firstException;
        }
    }

    public void failIfCorrupted() throws IOException {
        ensureOpen();
        failIfCorrupted(directory, shardId);
    }

    private static final void failIfCorrupted(Directory directory, ShardId shardId) throws IOException {
        final String[] files = directory.listAll();
        List<CorruptIndexException> ex = new ArrayList<>();
        for (String file : files) {
            if (file.startsWith(CORRUPTED)) {
                try (ChecksumIndexInput input = directory.openChecksumInput(file, IOContext.READONCE)) {
                    int version = CodecUtil.checkHeader(input, CODEC, VERSION_START, VERSION);
                    String msg = input.readString();
                    StringBuilder builder = new StringBuilder(shardId.toString());
                    builder.append(" Preexisting corrupted index [");
                    builder.append(file).append("] caused by: ");
                    builder.append(msg);
                    if (version == VERSION_STACK_TRACE) {
                        builder.append(System.lineSeparator());
                        builder.append(input.readString());
                    }
                    ex.add(new CorruptIndexException(builder.toString()));
                    CodecUtil.checkFooter(input);
                }
            }
        }
        if (ex.isEmpty() == false) {
            ExceptionsHelper.rethrowAndSuppress(ex);
        }
    }

    /**
     * This method deletes every file in this store that is not contained in the given source meta data or is a
     * legacy checksum file.
     *
     * @param reason the reason for this cleanup operation logged for each deleted file
     * @param filesToClean the file names used for cleanup. all files names in this metadata should be kept around.
     * @throws IOException if an IOException occurs
     */
    public void cleanup(String reason, Set<String> filesToClean) throws IOException {
        metadataLock.writeLock().lock();
        try (Lock writeLock = Lucene.acquireWriteLock(directory)) {
            final StoreDirectory dir = directory;
            for (String existingFile : dir.listAll()) {
                if (existingFile.equals(IndexWriter.WRITE_LOCK_NAME) || Store.isChecksum(existingFile) || filesToClean.contains(existingFile)) {
                    continue; // don't delete snapshot file, or the checksums file (note, this is extra protection since the Store won't delete checksum)
                }
                try {
                    logDeleteFile(reason, existingFile);
                    dir.deleteFile(existingFile);
                    // FNF should not happen since we hold a write lock?
                } catch (IOException ex) {
                    if (existingFile.startsWith(IndexFileNames.SEGMENTS)
                        || existingFile.equals(IndexFileNames.SEGMENTS_GEN)) {
                        // TODO do we need to also fail this if we can't delete the pending commit file?
                        // if one of those files can't be deleted we better fail the cleanup otherwise we might leave an old commit point around?
                        throw new ElasticsearchIllegalStateException("Can't delete " + existingFile + " - cleanup failed", ex);
                    }
                    logger.debug("failed to delete file [{}]", ex, existingFile);
                    // ignore, we don't really care, will get deleted later on
                }
            }
        } finally {
            metadataLock.writeLock().unlock();
        }
    }

    /**
     * This method deletes every file in this store that is not contained in the given source meta data or is a
     * legacy checksum file. After the delete it pulls the latest metadata snapshot from the store and compares it
     * to the given snapshot. If the snapshots are inconsistent an illegal state exception is thrown
     *
     * @param reason the reason for this cleanup operation logged for each deleted file
     * @param sourceMetaData the metadata used for cleanup. all files in this metadata should be kept around.
     * @throws IOException if an IOException occurs
     * @throws ElasticsearchIllegalStateException if the latest snapshot in this store differs from the given one after the cleanup.
     */
    public void cleanupAndVerify(String reason, MetadataSnapshot sourceMetaData) throws IOException {
        metadataLock.writeLock().lock();
        try {
            cleanup(reason, sourceMetaData.asMap().keySet());
            final Store.MetadataSnapshot metadataOrEmpty = getMetadata();
            verifyAfterCleanup(sourceMetaData, metadataOrEmpty);
        } finally {
            metadataLock.writeLock().unlock();
        }
    }

    // pkg private for testing
    final void verifyAfterCleanup(MetadataSnapshot sourceMetaData, MetadataSnapshot targetMetaData) {
        final RecoveryDiff recoveryDiff = targetMetaData.recoveryDiff(sourceMetaData);
        if (recoveryDiff.identical.size() != recoveryDiff.size()) {
            if (recoveryDiff.missing.isEmpty()) {
                for (StoreFileMetaData meta : recoveryDiff.different) {
                    StoreFileMetaData local = targetMetaData.get(meta.name());
                    StoreFileMetaData remote = sourceMetaData.get(meta.name());
                    // if we have different files the they must have no checksums otherwise something went wrong during recovery.
                    // we have that problem when we have an empty index is only a segments_1 file then we can't tell if it's a Lucene 4.8 file
                    // and therefore no checksum. That isn't much of a problem since we simply copy it over anyway but those files come out as
                    // different in the diff. That's why we have to double check here again if the rest of it matches.

                    // all is fine this file is just part of a commit or a segment that is different
                    final boolean same = local.isSame(remote);

                    // this check ensures that the two files are consistent ie. if we don't have checksums only the rest needs to match we are just
                    // verifying that we are consistent on both ends source and target
                    final boolean hashAndLengthEqual = (
                            local.checksum() == null
                                    && remote.checksum() == null
                                    && local.hash().equals(remote.hash())
                                    && local.length() == remote.length());
                    final boolean consistent = hashAndLengthEqual || same;
                    if (consistent == false) {
                        logger.debug("Files are different on the recovery target: {} ", recoveryDiff);
                        throw new ElasticsearchIllegalStateException("local version: " + local + " is different from remote version after recovery: " + remote, null);
                    }
                }
            } else {
                logger.debug("Files are missing on the recovery target: {} ", recoveryDiff);
                throw new ElasticsearchIllegalStateException("Files are missing on the recovery target: [different="
                        + recoveryDiff.different + ", missing=" + recoveryDiff.missing + ']', null);
            }
        }
    }

    /**
     * Returns the current reference count.
     */
    public int refCount() {
        return refCounter.refCount();
    }

    /**
     * This exists so {@link org.elasticsearch.index.codec.postingsformat.BloomFilterPostingsFormat} can load its boolean setting; can we find a more straightforward way?
     */
    public final class StoreDirectory extends FilterDirectory {

        public final ESLogger deletesLogger;

        StoreDirectory(Directory delegateDirectory) throws IOException {
            super(delegateDirectory);
            deletesLogger = Loggers.getLogger("index.store.deletes", indexSettings, shardId);
        }

        public ShardId shardId() {
            ensureOpen();
            return Store.this.shardId();
        }


        @Override
        public IndexInput openInput(String name, IOContext context) throws IOException {
            IndexInput in = super.openInput(name, context);
            boolean success = false;
            try {
                // Only for backward comp. since we now use Lucene codec compression
                if (name.endsWith(".fdt") || name.endsWith(".tvf")) {
                    Compressor compressor = CompressorFactory.compressor(in);
                    if (compressor != null) {
                        in = compressor.indexInput(in);
                    }
                }
                success = true;
            } finally {
                if (!success) {
                    IOUtils.closeWhileHandlingException(in);
                }
            }
            return in;
        }

        @Override
        public void close() throws IOException {
            assert false : "Nobody should close this directory except of the Store itself";
        }

        @Override
        public void deleteFile(String name) throws IOException {
            logDeleteFile("StoreDirectory.deleteFile", name);
            super.deleteFile(name);
        }

        private void innerClose() throws IOException {
            super.close();
        }

        @Override
        public String toString() {
            return "store(" + in.toString() + ")";
        }

    }

    /** Log that we are about to delete this file, to the index.store.deletes component. */
    public void logDeleteFile(String message, String fileName) {
        logDeleteFile(directory(), message, fileName);
    }

    /** Log that we are about to delete this file, to the index.store.deletes component. */
    public static void logDeleteFile(Directory dir, String message, String fileName) {
        assert dir instanceof StoreDirectory;
        if (dir instanceof StoreDirectory) {
            ((StoreDirectory) dir).deletesLogger.trace("{}: delete file {}", message, fileName);
        }
        // else what to do...?
    }

    /**
     * Represents a snapshot of the current directory build from the latest Lucene commit.
     * Only files that are part of the last commit are considered in this datastrucutre.
     * For backwards compatibility the snapshot might include legacy checksums that
     * are derived from a dedicated checksum file written by older elasticsearch version pre 1.3
     * <p/>
     * Note: This class will ignore the <tt>segments.gen</tt> file since it's optional and might
     * change concurrently for safety reasons.
     *
     * @see StoreFileMetaData
     */
    public final static class MetadataSnapshot implements Iterable<StoreFileMetaData>, Streamable {
        private static final ESLogger logger = Loggers.getLogger(MetadataSnapshot.class);
        private static final Version FIRST_LUCENE_CHECKSUM_VERSION = Version.LUCENE_48;
        // we stopped writing legacy checksums in 1.3.0 so all segments here must use the new CRC32 version
        private static final Version FIRST_ES_CRC32_VERSION = org.elasticsearch.Version.V_1_3_0.luceneVersion;

        private ImmutableMap<String, StoreFileMetaData> metadata;

        public static final MetadataSnapshot EMPTY = new MetadataSnapshot();

        private ImmutableMap<String, String> commitUserData;

        private long numDocs;

        public MetadataSnapshot(Map<String, StoreFileMetaData> metadata, Map<String, String> commitUserData, long numDocs) {
            ImmutableMap.Builder<String, StoreFileMetaData> metaDataBuilder = ImmutableMap.builder();
            this.metadata = metaDataBuilder.putAll(metadata).build();
            ImmutableMap.Builder<String, String> commitUserDataBuilder = ImmutableMap.builder();
            this.commitUserData = commitUserDataBuilder.putAll(commitUserData).build();
            this.numDocs = numDocs;
        }

        MetadataSnapshot() {
            metadata = ImmutableMap.of();
            commitUserData = ImmutableMap.of();
            numDocs = 0;
        }

        MetadataSnapshot(IndexCommit commit, Directory directory, ESLogger logger) throws IOException {
            LoadedMetadata loadedMetadata = loadMetadata(commit, directory, logger);
            metadata = loadedMetadata.fileMetadata;
            commitUserData = loadedMetadata.userData;
            numDocs = loadedMetadata.numDocs;
            assert metadata.isEmpty() || numSegmentFiles() == 1 : "numSegmentFiles: " + numSegmentFiles();
        }

        public MetadataSnapshot(StreamInput input) throws IOException {
            readFrom(input);
        }

        private static final boolean useLuceneChecksum(Version version, boolean hasLegacyChecksum) {
            return (version.onOrAfter(FIRST_LUCENE_CHECKSUM_VERSION) && hasLegacyChecksum == false) // no legacy checksum and a guarantee that lucene has checksums
                    || version.onOrAfter(FIRST_ES_CRC32_VERSION); // OR we know that we didn't even write legacy checksums anymore when this segment was written.
        }

        /**
         * Returns the number of documents in this store snapshot
         */
        public long getNumDocs() {
            return numDocs;
        }

        static class LoadedMetadata {
            final ImmutableMap<String, StoreFileMetaData> fileMetadata;
            final ImmutableMap<String, String> userData;
            final long numDocs;

            LoadedMetadata(ImmutableMap<String, StoreFileMetaData> fileMetadata, ImmutableMap<String, String> userData, long numDocs) {
                this.fileMetadata = fileMetadata;
                this.userData = userData;
                this.numDocs = numDocs;
            }
        }

        static LoadedMetadata loadMetadata(IndexCommit commit, Directory directory, ESLogger logger) throws IOException {
            long numDocs;
            ImmutableMap.Builder<String, StoreFileMetaData> builder = ImmutableMap.builder();
            Map<String, String> checksumMap = readLegacyChecksums(directory).v1();
            ImmutableMap.Builder<String, String> commitUserDataBuilder = ImmutableMap.builder();
            try {
                final SegmentInfos segmentCommitInfos = Store.readSegmentsInfo(commit, directory);
                Version maxVersion = Version.LUCENE_3_0; // we don't know which version was used to write so we take the max version.
                numDocs = Lucene.getNumDocs(segmentCommitInfos);
                commitUserDataBuilder.putAll(segmentCommitInfos.getUserData());
                for (SegmentCommitInfo info : segmentCommitInfos) {
                    final Version version = info.info.getVersion();
                    if (version != null && version.onOrAfter(maxVersion)) {
                        maxVersion = version;
                    }
                    for (String file : info.files()) {
                        String legacyChecksum = checksumMap.get(file);
                        if (useLuceneChecksum(version, legacyChecksum != null)) {
                            checksumFromLuceneFile(directory, file, builder, logger, version, Lucene46SegmentInfoFormat.SI_EXTENSION.equals(IndexFileNames.getExtension(file)));
                        } else {
                            builder.put(file, new StoreFileMetaData(file, directory.fileLength(file), legacyChecksum, null));
                        }
                    }
                }
                final String segmentsFile = segmentCommitInfos.getSegmentsFileName();
                String legacyChecksum = checksumMap.get(segmentsFile);
                if (useLuceneChecksum(maxVersion, legacyChecksum != null)) {
                    checksumFromLuceneFile(directory, segmentsFile, builder, logger, maxVersion, true);
                } else {
                    final BytesRefBuilder fileHash = new BytesRefBuilder();
                    final long length;
                    try (final IndexInput in = directory.openInput(segmentsFile, IOContext.READONCE)) {
                        length = in.length();
                        hashFile(fileHash, new InputStreamIndexInput(in, length), length);
                    }
                    builder.put(segmentsFile, new StoreFileMetaData(segmentsFile, length, legacyChecksum, null, fileHash.get()));
                }
            } catch (CorruptIndexException | IndexNotFoundException ex) {
                // we either know the index is corrupted or it's just not there
                throw ex;
            } catch (Throwable ex) {
                try {
                    // Lucene checks the checksum after it tries to lookup the codec etc.
                    // in that case we might get only IAE or similar exceptions while we are really corrupt...
                    // TODO we should check the checksum in lucene if we hit an exception
                    logger.warn("failed to build store metadata. checking segment info integrity (with commit [{}])",
                            ex, commit == null ? "no" : "yes");
                    Lucene.checkSegmentInfoIntegrity(directory);
                } catch (CorruptIndexException cex) {
                  cex.addSuppressed(ex);
                  throw cex;
                } catch (Throwable e) {
                    // ignore...
                }

                throw ex;
            }
            return new LoadedMetadata(builder.build(), commitUserDataBuilder.build(), numDocs);
        }

        /**
         * Reads legacy checksum files found in the directory.
         * <p/>
         * Files are expected to start with _checksums- prefix
         * followed by long file version. Only file with the highest version is read, all other files are ignored.
         *
         * @param directory the directory to read checksums from
         * @return a map of file checksums and the checksum file version
         * @throws IOException
         */
        static Tuple<Map<String, String>, Long> readLegacyChecksums(Directory directory) throws IOException {
            synchronized (directory) {
                long lastFound = -1;
                for (String name : directory.listAll()) {
                    if (!isChecksum(name)) {
                        continue;
                    }
                    long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length()));
                    if (current > lastFound) {
                        lastFound = current;
                    }
                }
                if (lastFound > -1) {
                    try (IndexInput indexInput = directory.openInput(CHECKSUMS_PREFIX + lastFound, IOContext.READONCE)) {
                        indexInput.readInt(); // version
                        return new Tuple(indexInput.readStringStringMap(), lastFound);
                    }
                }
                return new Tuple(new HashMap<>(), -1l);
            }
        }

        /**
         * Deletes all checksum files with version lower than newVersion.
         *
         * @param directory  the directory to clean
         * @param newVersion the latest checksum file version
         * @throws IOException
         */
        static void cleanLegacyChecksums(Directory directory, long newVersion) throws IOException {
            synchronized (directory) {
                for (String name : directory.listAll()) {
                    if (isChecksum(name)) {
                        long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length()));
                        if (current < newVersion) {
                            try {
                                directory.deleteFile(name);
                            } catch (IOException ex) {
                                logger.debug("can't delete old checksum file [{}]", ex, name);
                            }
                        }
                    }
                }
            }
        }

        private static void checksumFromLuceneFile(Directory directory, String file, ImmutableMap.Builder<String, StoreFileMetaData> builder, ESLogger logger, Version version, boolean readFileAsHash) throws IOException {
            final String checksum;
            final BytesRefBuilder fileHash = new BytesRefBuilder();
            try (final IndexInput in = directory.openInput(file, IOContext.READONCE)) {
                final long length;
                try {
                    length = in.length();
                    if (length < CodecUtil.footerLength()) {
                        // truncated files trigger IAE if we seek negative... these files are really corrupted though
                        throw new CorruptIndexException("Can't retrieve checksum from file: " + file + " file length must be >= " + CodecUtil.footerLength() + " but was: " + in.length());
                    }
                    if (readFileAsHash) {
                        final VerifyingIndexInput verifyingIndexInput = new VerifyingIndexInput(in); // additional safety we checksum the entire file we read the hash for...
                        hashFile(fileHash, new InputStreamIndexInput(verifyingIndexInput, length), length);
                        checksum = digestToString(verifyingIndexInput.verify());
                    } else {
                        checksum = digestToString(CodecUtil.retrieveChecksum(in));
                    }

                } catch (Throwable ex) {
                    logger.debug("Can retrieve checksum from file [{}]", ex, file);
                    throw ex;
                }
                builder.put(file, new StoreFileMetaData(file, length, checksum, version, fileHash.get()));
            }
        }

        /**
         * Computes a strong hash value for small files. Note that this method should only be used for files < 1MB
         */
        public static BytesRef hashFile(Directory directory, String file) throws IOException {
            final BytesRefBuilder fileHash = new BytesRefBuilder();
            try (final IndexInput in = directory.openInput(file, IOContext.READONCE)) {
                hashFile(fileHash, new InputStreamIndexInput(in, in.length()), in.length());
            }
            return fileHash.get();
        }


        /**
         * Computes a strong hash value for small files. Note that this method should only be used for files < 1MB
         */
        public static void hashFile(BytesRefBuilder fileHash, InputStream in, long size) throws IOException {
            final int len = (int) Math.min(1024 * 1024, size); // for safety we limit this to 1MB
            fileHash.grow(len);
            fileHash.setLength(len);
            final int readBytes = Streams.readFully(in, fileHash.bytes(), 0, len);
            assert readBytes == len : Integer.toString(readBytes) + " != " + Integer.toString(len);
            assert fileHash.length() == len : Integer.toString(fileHash.length()) + " != " + Integer.toString(len);
        }

        @Override
        public Iterator<StoreFileMetaData> iterator() {
            return metadata.values().iterator();
        }

        public StoreFileMetaData get(String name) {
            return metadata.get(name);
        }

        public Map<String, StoreFileMetaData> asMap() {
            return metadata;
        }

        private static final String DEL_FILE_EXTENSION = "del";  // TODO think about how we can detect if this changes?
        private static final String FIELD_INFOS_FILE_EXTENSION = "fnm";

        /**
         * Returns a diff between the two snapshots that can be used for recovery. The given snapshot is treated as the
         * recovery target and this snapshot as the source. The returned diff will hold a list of files that are:
         * <ul>
         * <li>identical: they exist in both snapshots and they can be considered the same ie. they don't need to be recovered</li>
         * <li>different: they exist in both snapshots but their they are not identical</li>
         * <li>missing: files that exist in the source but not in the target</li>
         * </ul>
         * This method groups file into per-segment files and per-commit files. A file is treated as
         * identical if and on if all files in it's group are identical. On a per-segment level files for a segment are treated
         * as identical iff:
         * <ul>
         * <li>all files in this segment have the same checksum</li>
         * <li>all files in this segment have the same length</li>
         * <li>the segments <tt>.si</tt> files hashes are byte-identical Note: This is a using a perfect hash function, The metadata transfers the <tt>.si</tt> file content as it's hash</li>
         * </ul>
         * <p/>
         * The <tt>.si</tt> file contains a lot of diagnostics including a timestamp etc. in the future there might be
         * unique segment identifiers in there hardening this method further.
         * <p/>
         * The per-commit files handles very similar. A commit is composed of the <tt>segments_N</tt> files as well as generational files like
         * deletes (<tt>_x_y.del</tt>) or field-info (<tt>_x_y.fnm</tt>) files. On a per-commit level files for a commit are treated
         * as identical iff:
         * <ul>
         * <li>all files belonging to this commit have the same checksum</li>
         * <li>all files belonging to this commit have the same length</li>
         * <li>the segments file <tt>segments_N</tt> files hashes are byte-identical Note: This is a using a perfect hash function, The metadata transfers the <tt>segments_N</tt> file content as it's hash</li>
         * </ul>
         * <p/>
         * NOTE: this diff will not contain the <tt>segments.gen</tt> file. This file is omitted on recovery.
         */
        public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) {
            final ImmutableList.Builder<StoreFileMetaData> identical = ImmutableList.builder();
            final ImmutableList.Builder<StoreFileMetaData> different = ImmutableList.builder();
            final ImmutableList.Builder<StoreFileMetaData> missing = ImmutableList.builder();
            final Map<String, List<StoreFileMetaData>> perSegment = new HashMap<>();
            final List<StoreFileMetaData> perCommitStoreFiles = new ArrayList<>();

            for (StoreFileMetaData meta : this) {
                if (IndexFileNames.SEGMENTS_GEN.equals(meta.name())) {
                    continue; // we don't need that file at all
                }
                final String segmentId = IndexFileNames.parseSegmentName(meta.name());
                final String extension = IndexFileNames.getExtension(meta.name());
                assert FIELD_INFOS_FILE_EXTENSION.equals(extension) == false || IndexFileNames.stripExtension(IndexFileNames.stripSegmentName(meta.name())).isEmpty() : "FieldInfos are generational but updateable DV are not supported in elasticsearch";
                if (IndexFileNames.SEGMENTS.equals(segmentId) || DEL_FILE_EXTENSION.equals(extension)) {
                        // only treat del files as per-commit files fnm files are generational but only for upgradable DV
                    perCommitStoreFiles.add(meta);
                } else {
                    List<StoreFileMetaData> perSegStoreFiles = perSegment.get(segmentId);
                    if (perSegStoreFiles == null) {
                        perSegStoreFiles = new ArrayList<>();
                        perSegment.put(segmentId, perSegStoreFiles);
                    }
                    perSegStoreFiles.add(meta);
                }
            }
            final ArrayList<StoreFileMetaData> identicalFiles = new ArrayList<>();
            for (List<StoreFileMetaData> segmentFiles : Iterables.concat(perSegment.values(), Collections.singleton(perCommitStoreFiles))) {
                identicalFiles.clear();
                boolean consistent = true;
                for (StoreFileMetaData meta : segmentFiles) {
                    StoreFileMetaData storeFileMetaData = recoveryTargetSnapshot.get(meta.name());
                    if (storeFileMetaData == null) {
                        consistent = false;
                        missing.add(meta);
                    } else if (storeFileMetaData.isSame(meta) == false) {
                        consistent = false;
                        different.add(meta);
                    } else {
                        identicalFiles.add(meta);
                    }
                }
                if (consistent) {
                    identical.addAll(identicalFiles);
                } else {
                    // make sure all files are added - this can happen if only the deletes are different
                    different.addAll(identicalFiles);
                }
            }
            RecoveryDiff recoveryDiff = new RecoveryDiff(identical.build(), different.build(), missing.build());
            assert recoveryDiff.size() == this.metadata.size() - (metadata.containsKey(IndexFileNames.SEGMENTS_GEN) ? 1: 0)
                    : "some files are missing recoveryDiff size: [" + recoveryDiff.size() + "] metadata size: [" + this.metadata.size()  + "] contains  segments.gen: [" + metadata.containsKey(IndexFileNames.SEGMENTS_GEN) + "]"   ;
            return recoveryDiff;
        }

        /**
         * Returns the number of files in this snapshot
         */
        public int size() {
            return metadata.size();
        }

        @Override
        public void writeTo(StreamOutput out) throws IOException {
            out.writeVInt(this.metadata.size());
            for (StoreFileMetaData meta : this) {
                meta.writeTo(out);
            }
            if (out.getVersion().onOrAfter(org.elasticsearch.Version.V_1_6_0)) {
                out.writeVInt(commitUserData.size());
                for (Map.Entry<String, String> entry : commitUserData.entrySet()) {
                    out.writeString(entry.getKey());
                    out.writeString(entry.getValue());
                }
                out.writeLong(numDocs);
            }
        }

        public Map<String, String> getCommitUserData() {
            return commitUserData;
        }

        /**
         * Returns true iff this metadata contains the given file.
         */
        public boolean contains(String existingFile) {
            return metadata.containsKey(existingFile);
        }

        /**
         * Returns the segments file that this metadata snapshot represents or null if the snapshot is empty.
         */
        public StoreFileMetaData getSegmentsFile() {
            for (StoreFileMetaData file : this) {
                if (file.name().startsWith(IndexFileNames.SEGMENTS)) {
                    return file;
                }
            }
            assert metadata.isEmpty();
            return null;
        }

        private final int numSegmentFiles() { // only for asserts
            int count = 0;
            for (StoreFileMetaData file : this) {
                if (file.name().startsWith(IndexFileNames.SEGMENTS)) {
                    count++;
                }
            }
            return count;
        }

        /**
         * Returns the sync id of the commit point that this MetadataSnapshot represents.
         *
         * @return sync id if exists, else null
         */
        public String getSyncId() {
            return commitUserData.get(Engine.SYNC_COMMIT_ID);
        }

        @Override
        public void readFrom(StreamInput in) throws IOException {
            final int size = in.readVInt();
            final ImmutableMap.Builder<String, StoreFileMetaData> metadataBuilder = ImmutableMap.builder();
            for (int i = 0; i < size; i++) {
                StoreFileMetaData meta = StoreFileMetaData.readStoreFileMetaData(in);
                metadataBuilder.put(meta.name(), meta);
            }
            this.metadata = metadataBuilder.build();
            final ImmutableMap.Builder<String, String> commitUserDataBuilder = ImmutableMap.builder();
            if (in.getVersion().onOrAfter(org.elasticsearch.Version.V_1_6_0)) {
                int num = in.readVInt();
                for (int i = num; i > 0; i--) {
                    commitUserDataBuilder.put(in.readString(), in.readString());
                }

                this.numDocs = in.readLong();
                assert metadata.isEmpty() || numSegmentFiles() == 1 : "numSegmentFiles: " + numSegmentFiles();
            } else {
                this.numDocs = -1;
            }
            this.commitUserData = commitUserDataBuilder.build();
        }
    }

    /**
     * A class representing the diff between a recovery source and recovery target
     *
     * @see MetadataSnapshot#recoveryDiff(org.elasticsearch.index.store.Store.MetadataSnapshot)
     */
    public static final class RecoveryDiff {
        /**
         * Files that exist in both snapshots and they can be considered the same ie. they don't need to be recovered
         */
        public final List<StoreFileMetaData> identical;
        /**
         * Files that exist in both snapshots but their they are not identical
         */
        public final List<StoreFileMetaData> different;
        /**
         * Files that exist in the source but not in the target
         */
        public final List<StoreFileMetaData> missing;

        RecoveryDiff(List<StoreFileMetaData> identical, List<StoreFileMetaData> different, List<StoreFileMetaData> missing) {
            this.identical = identical;
            this.different = different;
            this.missing = missing;
        }

        /**
         * Returns the sum of the files in this diff.
         */
        public int size() {
            return identical.size() + different.size() + missing.size();
        }

        @Override
        public String toString() {
            return "RecoveryDiff{" +
                    "identical=" + identical +
                    ", different=" + different +
                    ", missing=" + missing +
                    '}';
        }
    }

    public final static class LegacyChecksums {
        private final Map<String, String> legacyChecksums = new HashMap<>();

        public void add(StoreFileMetaData metaData) throws IOException {

            if (metaData.hasLegacyChecksum()) {
                synchronized (this) {
                    // we don't add checksums if they were written by LUCENE_48... now we are using the build in mechanism.
                    legacyChecksums.put(metaData.name(), metaData.checksum());
                }
            }
        }

        public synchronized void write(Store store) throws IOException {
            synchronized (store.directory) {
                Tuple<Map<String, String>, Long> tuple = MetadataSnapshot.readLegacyChecksums(store.directory);
                tuple.v1().putAll(legacyChecksums);
                if (!tuple.v1().isEmpty()) {
                    writeChecksums(store.directory, tuple.v1(), tuple.v2());
                }
            }
        }

        synchronized void writeChecksums(Directory directory, Map<String, String> checksums, long lastVersion) throws IOException {
            // Make sure if clock goes backwards we still move version forwards:
            long nextVersion = Math.max(lastVersion+1, System.currentTimeMillis());
            final String checksumName = CHECKSUMS_PREFIX + nextVersion;
            try (IndexOutput output = directory.createOutput(checksumName, IOContext.DEFAULT)) {
                output.writeInt(0); // version
                output.writeStringStringMap(checksums);
            }
            directory.sync(Collections.singleton(checksumName));
            MetadataSnapshot.cleanLegacyChecksums(directory, nextVersion);
        }

        public void clear() {
            this.legacyChecksums.clear();
        }

        public void remove(String name) {
            legacyChecksums.remove(name);
        }
    }

    public static final String CHECKSUMS_PREFIX = "_checksums-";

    public static final boolean isChecksum(String name) {
        // TODO can we drowp .cks
        return name.startsWith(CHECKSUMS_PREFIX) || name.endsWith(".cks"); // bwcomapt - .cks used to be a previous checksum file
    }

    /**
     * Produces a string representation of the given digest value.
     */
    public static String digestToString(long digest) {
        return Long.toString(digest, Character.MAX_RADIX);
    }


    static class LuceneVerifyingIndexOutput extends VerifyingIndexOutput {

        private final StoreFileMetaData metadata;
        private long writtenBytes;
        private final long checksumPosition;
        private String actualChecksum;

        LuceneVerifyingIndexOutput(StoreFileMetaData metadata, IndexOutput out) {
            super(out, metadata.name());
            this.metadata = metadata;
            checksumPosition = metadata.length() - 8; // the last 8 bytes are the checksum
        }

        @Override
        public void verify() throws IOException {
            if (metadata.checksum().equals(actualChecksum) && writtenBytes == metadata.length()) {
                return;
            }
            throw new CorruptIndexException("verification failed (hardware problem?) : expected=" + metadata.checksum() +
                    " actual=" + actualChecksum + " writtenLength=" + writtenBytes + " expectedLength=" + metadata.length() +
                    " (resource=" + metadata.toString() + ")");
        }

        @Override
        public void writeByte(byte b) throws IOException {
            if (writtenBytes++ == checksumPosition) {
                readAndCompareChecksum();
            }
            out.writeByte(b);
        }

        private void readAndCompareChecksum() throws IOException {
            actualChecksum = digestToString(getChecksum());
            if (!metadata.checksum().equals(actualChecksum)) {
                throw new CorruptIndexException("checksum failed (hardware problem?) : expected=" + metadata.checksum() +
                        " actual=" + actualChecksum +
                        " (resource=" + metadata.toString() + ")");
            }
        }

        @Override
        public void writeBytes(byte[] b, int offset, int length) throws IOException {
            if (writtenBytes + length > checksumPosition && actualChecksum == null) {
                assert writtenBytes <= checksumPosition;
                final int bytesToWrite = (int) (checksumPosition - writtenBytes);
                out.writeBytes(b, offset, bytesToWrite);
                readAndCompareChecksum();
                offset += bytesToWrite;
                length -= bytesToWrite;
                writtenBytes += bytesToWrite;
            }
            out.writeBytes(b, offset, length);
            writtenBytes += length;
        }

    }

    /**
     * Index input that calculates checksum as data is read from the input.
     * <p/>
     * This class supports random access (it is possible to seek backward and forward) in order to accommodate retry
     * mechanism that is used in some repository plugins (S3 for example). However, the checksum is only calculated on
     * the first read. All consecutive reads of the same data are not used to calculate the checksum.
     */
    static class VerifyingIndexInput extends ChecksumIndexInput {
        private final IndexInput input;
        private final Checksum digest;
        private final long checksumPosition;
        private final byte[] checksum = new byte[8];
        private long verifiedPosition = 0;

        public VerifyingIndexInput(IndexInput input) {
            this(input, new BufferedChecksum(new CRC32()));
        }

        public VerifyingIndexInput(IndexInput input, Checksum digest) {
            super("VerifyingIndexInput(" + input + ")");
            this.input = input;
            this.digest = digest;
            checksumPosition = input.length() - 8;
        }

        @Override
        public byte readByte() throws IOException {
            long pos = input.getFilePointer();
            final byte b = input.readByte();
            pos++;
            if (pos > verifiedPosition) {
                if (pos <= checksumPosition) {
                    digest.update(b);
                } else {
                    checksum[(int) (pos - checksumPosition - 1)] = b;
                }
                verifiedPosition = pos;
            }
            return b;
        }

        @Override
        public void readBytes(byte[] b, int offset, int len)
                throws IOException {
            long pos = input.getFilePointer();
            input.readBytes(b, offset, len);
            if (pos + len > verifiedPosition) {
                // Conversion to int is safe here because (verifiedPosition - pos) can be at most len, which is integer
                int alreadyVerified = (int) Math.max(0, verifiedPosition - pos);
                if (pos < checksumPosition) {
                    if (pos + len < checksumPosition) {
                        digest.update(b, offset + alreadyVerified, len - alreadyVerified);
                    } else {
                        int checksumOffset = (int) (checksumPosition - pos);
                        if (checksumOffset - alreadyVerified > 0) {
                            digest.update(b, offset + alreadyVerified, checksumOffset - alreadyVerified);
                        }
                        System.arraycopy(b, offset + checksumOffset, checksum, 0, len - checksumOffset);
                    }
                } else {
                    // Conversion to int is safe here because checksumPosition is (file length - 8) so
                    // (pos - checksumPosition) cannot be bigger than 8 unless we are reading after the end of file
                    assert pos - checksumPosition < 8;
                    System.arraycopy(b, offset, checksum, (int) (pos - checksumPosition), len);
                }
                verifiedPosition = pos + len;
            }
        }

        @Override
        public long getChecksum() {
            return digest.getValue();
        }

        @Override
        public void seek(long pos) throws IOException {
            if (pos < verifiedPosition) {
                // going within verified region - just seek there
                input.seek(pos);
            } else {
                if (verifiedPosition > getFilePointer()) {
                    // portion of the skip region is verified and portion is not
                    // skipping the verified portion
                    input.seek(verifiedPosition);
                    // and checking unverified
                    skipBytes(pos - verifiedPosition);
                } else {
                    skipBytes(pos - getFilePointer());
                }
            }
        }

        @Override
        public void close() throws IOException {
            input.close();
        }

        @Override
        public long getFilePointer() {
            return input.getFilePointer();
        }

        @Override
        public long length() {
            return input.length();
        }

        @Override
        public IndexInput clone() {
            throw new UnsupportedOperationException();
        }

        @Override
        public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
            throw new UnsupportedOperationException();
        }

        public long getStoredChecksum() {
            return new ByteArrayDataInput(checksum).readLong();
        }

        public long verify() throws CorruptIndexException {
            long storedChecksum = getStoredChecksum();
            if (getChecksum() == storedChecksum) {
                return storedChecksum;
            }
            throw new CorruptIndexException("verification failed : calculated=" + Store.digestToString(getChecksum()) +
                    " stored=" + Store.digestToString(storedChecksum));
        }

    }

    public void deleteQuiet(String... files) {
        for (String file : files) {
            try {
                directory().deleteFile(file);
            } catch (Throwable ex) {
                // ignore
            }
        }
    }

    /**
     * Marks this store as corrupted. This method writes a <tt>corrupted_${uuid}</tt> file containing the given exception
     * message. If a store contains a <tt>corrupted_${uuid}</tt> file {@link #isMarkedCorrupted()} will return <code>true</code>.
     */
    public void markStoreCorrupted(IOException exception) throws IOException {
        ensureOpen();
        if (!isMarkedCorrupted()) {
            String uuid = CORRUPTED + Strings.randomBase64UUID();
            try (IndexOutput output = this.directory().createOutput(uuid, IOContext.DEFAULT)) {
                CodecUtil.writeHeader(output, CODEC, VERSION);
                output.writeString(ExceptionsHelper.detailedMessage(exception, true, 0)); // handles null exception
                output.writeString(ExceptionsHelper.stackTrace(exception));
                CodecUtil.writeFooter(output);
            } catch (IOException ex) {
                logger.warn("Can't mark store as corrupted", ex);
            }
            directory().sync(Collections.singleton(uuid));
        }
    }

    /**
     * A listener that is executed once the store is closed and all references to it are released
     */
    public static interface OnClose extends Callback<ShardLock> {
        static final OnClose EMPTY = new OnClose() {
            /**
             * This method is called while the provided {@link org.elasticsearch.env.ShardLock} is held.
             * This method is only called once after all resources for a store are released.
             */
            @Override
            public void handle(ShardLock Lock) {
            }
        };
    }

    private static class StoreStatsCache extends SingleObjectCache<StoreStats> {
        private final Directory directory;
        private final DirectoryService directoryService;

        public StoreStatsCache(TimeValue refreshInterval, Directory directory, DirectoryService directoryService) throws IOException {
            super(refreshInterval, new StoreStats(estimateSize(directory), directoryService.throttleTimeInNanos()));
            this.directory = directory;
            this.directoryService = directoryService;
        }

        @Override
        protected StoreStats refresh() {
            try {
                return new StoreStats(estimateSize(directory), directoryService.throttleTimeInNanos());
            } catch (IOException ex) {
                throw new ElasticsearchException("failed to refresh store stats");
            }
        }

        private static long estimateSize(Directory directory) throws IOException {
            long estimatedSize = 0;
            String[] files = directory.listAll();
            for (String file : files) {
                try {
                    estimatedSize += directory.fileLength(file);
                } catch (NoSuchFileException | FileNotFoundException e) {
                    // ignore, the file is not there no more
                }
            }
            return estimatedSize;
        }
    }
}
