# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import copy
import cStringIO
import unittest
import tempfile
import json
import os

import mock

from core import perf_data_generator
from core.perf_data_generator import BenchmarkMetadata


class PerfDataGeneratorTest(unittest.TestCase):
  def setUp(self):
    # Test config can be big, so set maxDiff to None to see the full comparision
    # diff when assertEquals fails.
    self.maxDiff = None

  def test_get_scheduled_non_telemetry_benchmarks(self):
    tmpfile = tempfile.NamedTemporaryFile(delete=False)
    tmpfile.close()
    fake_perf_waterfall_file = tmpfile.name

    data = {
        'builder 1': {
          'isolated_scripts': [
            {'name': 'test_dancing'},
            {'name': 'test_singing'},
            {'name': 'performance_test_suite'},
          ],
          'scripts': [
            {'name': 'ninja_test'},
          ]
        },
        'builder 2': {
          'scripts': [
            {'name': 'gun_slinger'},
          ]
        }
      }
    try:
      with open(fake_perf_waterfall_file, 'w') as f:
        json.dump(data, f)
      self.assertEquals(
          perf_data_generator.get_scheduled_non_telemetry_benchmarks(
              fake_perf_waterfall_file),
          {'ninja_test', 'gun_slinger', 'test_dancing', 'test_singing'})
    finally:
      os.remove(fake_perf_waterfall_file)


  def testGenerateCPlusPlusTestSuite(self):
    swarming_dimensions = [
        {'os': 'SkyNet', 'pool': 'T-RIP'}
    ]
    test_config = {
        'platform': 'win',
        'dimension': swarming_dimensions,
    }
    test = {
        'isolate': 'angle_perftest',
        'telemetry': False,
        'num_shards': 1
    }
    returned_test = perf_data_generator.generate_performance_test(
        test_config, test)

    expected_generated_test = {
        'override_compile_targets': ['angle_perftest'],
        'isolate_name': 'angle_perftest',
        'args': ['--gtest-benchmark-name', 'angle_perftest',
                 '--non-telemetry=true', '--migrated-test=true'],
        'trigger_script': {
          'args': [
            '--multiple-dimension-script-verbose',
            'True'
          ],
          'script': '//testing/trigger_scripts/perf_device_trigger.py'
        },
        'merge': {
          'script': '//tools/perf/process_perf_results.py'
        },
        'swarming': {
          'ignore_task_failure': False,
          'can_use_on_swarming_builders': True,
          'expiration': 7200,
          'io_timeout': 1800,
          'hard_timeout': 36000,
          'upload_test_results': True,
          'dimension_sets': [[{'os': 'SkyNet', 'pool': 'T-RIP'}]],
          'shards': 1
        },
        'name': 'angle_perftest'
      }
    self.assertEquals(returned_test, expected_generated_test)

  def testGeneratePerformanceTestSuiteWebview(self):
    swarming_dimensions = [
        {'os': 'SkyNet', 'pool': 'T-RIP'}
    ]
    test_config = {
        'platform': 'android-webview',
        'dimension': swarming_dimensions,
    }
    test = {
        'isolate': 'performance_test_suite',
        'extra_args': [
            '--run-ref-build',
            '--test-shard-map-filename=shard_map.json',
          ],
        'num_shards': 26
    }
    returned_test = perf_data_generator.generate_performance_test(
        test_config, test)

    expected_generated_test = {
        'override_compile_targets': ['performance_test_suite'],
        'isolate_name': 'performance_test_suite',
        'args': ['-v', '--browser=android-webview', '--upload-results',
                 '--webview-embedder-apk=../../out/Release/apks/SystemWebViewShell.apk',
                 '--run-ref-build',
                 '--test-shard-map-filename=shard_map.json'],
        'trigger_script': {
          'args': [
            '--multiple-dimension-script-verbose',
            'True'
          ],
          'script': '//testing/trigger_scripts/perf_device_trigger.py'
        },
        'merge': {
          'script': '//tools/perf/process_perf_results.py'
        },
        'swarming': {
          'ignore_task_failure': False,
          'can_use_on_swarming_builders': True,
          'expiration': 7200,
          'io_timeout': 1800,
          'hard_timeout': 36000,
          'upload_test_results': True,
          'dimension_sets': [[{'os': 'SkyNet', 'pool': 'T-RIP'}]],
          'shards': 26
        },
        'name': 'performance_test_suite'
      }
    self.assertEquals(returned_test, expected_generated_test)

  def testGeneratePerformanceTestSuite(self):
    swarming_dimensions = [
        {'os': 'SkyNet', 'pool': 'T-RIP'}
    ]
    test_config = {
        'platform': 'android',
        'dimension': swarming_dimensions,
    }
    test = {
        'isolate': 'performance_test_suite',
        'extra_args': [
            '--run-ref-build',
            '--test-shard-map-filename=shard_map.json',
          ],
        'num_shards': 26
    }
    returned_test = perf_data_generator.generate_performance_test(
        test_config, test)

    expected_generated_test = {
        'override_compile_targets': ['performance_test_suite'],
        'isolate_name': 'performance_test_suite',
        'args': ['-v', '--browser=android-chromium', '--upload-results',
                 '--run-ref-build',
                 '--test-shard-map-filename=shard_map.json'],
        'trigger_script': {
          'args': [
            '--multiple-dimension-script-verbose',
            'True'
          ],
          'script': '//testing/trigger_scripts/perf_device_trigger.py'
        },
        'merge': {
          'script': '//tools/perf/process_perf_results.py'
        },
        'swarming': {
          'ignore_task_failure': False,
          'can_use_on_swarming_builders': True,
          'expiration': 7200,
          'io_timeout': 1800,
          'hard_timeout': 36000,
          'upload_test_results': True,
          'dimension_sets': [[{'os': 'SkyNet', 'pool': 'T-RIP'}]],
          'shards': 26
        },
        'name': 'performance_test_suite'
      }
    self.assertEquals(returned_test, expected_generated_test)


class TestIsPerfBenchmarksSchedulingValid(unittest.TestCase):
  def setUp(self):
    self.maxDiff = None
    self.original_NON_TELEMETRY_BENCHMARKS = copy.deepcopy(
        perf_data_generator.NON_TELEMETRY_BENCHMARKS)
    self.original_TELEMETRY_PERF_BENCHMARKS = copy.deepcopy(
        perf_data_generator.NON_TELEMETRY_BENCHMARKS)
    self.test_stream = cStringIO.StringIO()
    self.mock_get_telemetry_benchmarks = mock.patch(
        'core.perf_data_generator.get_telemetry_tests_in_performance_test_suite'
        )
    self.get_telemetry_benchmarks = (
        self.mock_get_telemetry_benchmarks.start())
    self.mock_get_non_telemetry_benchmarks = mock.patch(
        'core.perf_data_generator.get_scheduled_non_telemetry_benchmarks')
    self.get_non_telemetry_benchmarks = (
        self.mock_get_non_telemetry_benchmarks.start())

  def tearDown(self):
    perf_data_generator.TELEMETRY_PERF_BENCHMARKS = (
        self.original_TELEMETRY_PERF_BENCHMARKS)
    perf_data_generator.NON_TELEMETRY_BENCHMARKS = (
        self.original_NON_TELEMETRY_BENCHMARKS)
    self.mock_get_telemetry_benchmarks.stop()
    self.mock_get_non_telemetry_benchmarks.stop()

  def test_returnTrue(self):
    self.get_telemetry_benchmarks.return_value = {'t_foo', 't_bar'}
    self.get_non_telemetry_benchmarks.return_value = {'honda'}

    perf_data_generator.TELEMETRY_PERF_BENCHMARKS = {
        't_foo': BenchmarkMetadata('t@foo.com'),
        't_bar': BenchmarkMetadata('t@bar.com'),
    }
    perf_data_generator.NON_TELEMETRY_BENCHMARKS = {
        'honda': BenchmarkMetadata('baz@foo.com'),
    }
    valid = perf_data_generator.is_perf_benchmarks_scheduling_valid(
        'dummy', self.test_stream)

    self.assertEquals(valid, True)
    self.assertEquals(self.test_stream.getvalue(), '')

  def test_UnscheduledTelemetryBenchmarks(
      self):
    self.get_telemetry_benchmarks.return_value = {'t_bar'}
    self.get_non_telemetry_benchmarks.return_value = {'honda'}

    perf_data_generator.TELEMETRY_PERF_BENCHMARKS = {
        'darth.vader': BenchmarkMetadata('death@star.com'),
        't_bar': BenchmarkMetadata('t@bar.com'),
    }
    perf_data_generator.NON_TELEMETRY_BENCHMARKS = {
        'honda': BenchmarkMetadata('baz@foo.com'),
    }
    valid = perf_data_generator.is_perf_benchmarks_scheduling_valid(
        'dummy', self.test_stream)

    self.assertEquals(valid, False)
    self.assertIn('Telemetry benchmark darth.vader exists but is not scheduled',
        self.test_stream.getvalue())

  def test_UnscheduledCppBenchmarks(self):
    self.get_telemetry_benchmarks.return_value = {'t_bar'}
    self.get_non_telemetry_benchmarks.return_value = {'honda'}

    perf_data_generator.TELEMETRY_PERF_BENCHMARKS = {
        't_bar': BenchmarkMetadata('t@bar.com'),
    }
    perf_data_generator.NON_TELEMETRY_BENCHMARKS = {
        'honda': BenchmarkMetadata('baz@foo.com'),
        'toyota': BenchmarkMetadata('baz@foo.com'),
    }
    valid = perf_data_generator.is_perf_benchmarks_scheduling_valid(
        'dummy', self.test_stream)

    self.assertEquals(valid, False)
    self.assertIn('Benchmark toyota is tracked but not scheduled',
        self.test_stream.getvalue())

  def test_UntrackedTelemetryBenchmarks(
      self):
    self.get_telemetry_benchmarks.return_value = {'t_bar', 'darth.vader'}
    self.get_non_telemetry_benchmarks.return_value = {'honda'}

    perf_data_generator.TELEMETRY_PERF_BENCHMARKS = {
        't_bar': BenchmarkMetadata('t@bar.com'),
    }
    perf_data_generator.NON_TELEMETRY_BENCHMARKS = {
        'honda': BenchmarkMetadata('baz@foo.com'),
    }
    valid = perf_data_generator.is_perf_benchmarks_scheduling_valid(
        'dummy', self.test_stream)

    self.assertEquals(valid, False)
    self.assertIn('Telemetry benchmark darth.vader no longer exists',
        self.test_stream.getvalue())

  def test_UntrackedCppBenchmarks(self):
    self.get_telemetry_benchmarks.return_value = {'t_bar'}
    self.get_non_telemetry_benchmarks.return_value = {'honda', 'tesla'}

    perf_data_generator.TELEMETRY_PERF_BENCHMARKS = {
        't_bar': BenchmarkMetadata('t@bar.com'),
    }
    perf_data_generator.NON_TELEMETRY_BENCHMARKS = {
        'honda': BenchmarkMetadata('baz@foo.com'),
    }
    valid = perf_data_generator.is_perf_benchmarks_scheduling_valid(
        'dummy', self.test_stream)

    self.assertEquals(valid, False)
    self.assertIn(
        'Benchmark tesla is scheduled on perf waterfall but not tracked',
        self.test_stream.getvalue())
