// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "net/http/transport_security_persister.h"

#include <memory>
#include <utility>

#include "base/base64.h"
#include "base/bind.h"
#include "base/feature_list.h"
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/json/json_reader.h"
#include "base/json/json_writer.h"
#include "base/location.h"
#include "base/metrics/histogram_macros.h"
#include "base/sequenced_task_runner.h"
#include "base/task_runner_util.h"
#include "base/threading/thread_task_runner_handle.h"
#include "base/values.h"
#include "crypto/sha2.h"
#include "net/base/features.h"
#include "net/base/network_isolation_key.h"
#include "net/cert/x509_certificate.h"
#include "net/http/transport_security_state.h"

namespace net {

namespace {

// This function converts the binary hashes to a base64 string which we can
// include in a JSON file.
std::string HashedDomainToExternalString(const std::string& hashed) {
  std::string out;
  base::Base64Encode(hashed, &out);
  return out;
}

// This inverts |HashedDomainToExternalString|, above. It turns an external
// string (from a JSON file) into an internal (binary) string.
std::string ExternalStringToHashedDomain(const std::string& external) {
  std::string out;
  if (!base::Base64Decode(external, &out) ||
      out.size() != crypto::kSHA256Length) {
    return std::string();
  }

  return out;
}

// Version 2 of the on-disk format consists of a single JSON object. The
// top-level dictionary has "version", "sts", and "expect_ct" entries. The first
// is an integer, the latter two are unordered lists of dictionaries, each
// representing cached data for a single host.

// Stored in serialized dictionary values to distinguish incompatible versions.
// Version 1 is distinguished by the lack of an integer version value.
const char kVersionKey[] = "version";
const int kCurrentVersionValue = 2;

// Keys in top level serialized dictionary, for lists of STS and Expect-CT
// entries, respectively.
const char kSTSKey[] = "sts";
const char kExpectCTKey[] = "expect_ct";

// Hostname entry, used in serialized STS and Expect-CT dictionaries. Value is
// produced by passing hashed hostname strings to
// HashedDomainToExternalString().
const char kHostname[] = "host";

// Key values in serialized STS entries.
const char kStsIncludeSubdomains[] = "sts_include_subdomains";
const char kStsObserved[] = "sts_observed";
const char kExpiry[] = "expiry";
const char kMode[] = "mode";

// Values for "mode" used in serialized STS entries.
const char kForceHTTPS[] = "force-https";
const char kDefault[] = "default";

// Key names in serialized Expect-CT entries.
const char kNetworkIsolationKey[] = "nik";
const char kExpectCTObserved[] = "expect_ct_observed";
const char kExpectCTExpiry[] = "expect_ct_expiry";
const char kExpectCTEnforce[] = "expect_ct_enforce";
const char kExpectCTReportUri[] = "expect_ct_report_uri";

// Obsolete values in older STS entries.
const char kIncludeSubdomains[] = "include_subdomains";
const char kStrict[] = "strict";
const char kPinningOnly[] = "pinning-only";
const char kCreated[] = "created";
const char kExpectCTSubdictionary[] = "expect_ct";

std::string LoadState(const base::FilePath& path) {
  std::string result;
  if (!base::ReadFileToString(path, &result)) {
    return "";
  }
  return result;
}

bool IsDynamicExpectCTEnabled() {
  return base::FeatureList::IsEnabled(
      TransportSecurityState::kDynamicExpectCTFeature);
}

// Serializes STS data from |state| to a Value.
base::Value SerializeSTSData(const TransportSecurityState* state) {
  base::Value sts_list(base::Value::Type::LIST);

  TransportSecurityState::STSStateIterator sts_iterator(*state);
  for (; sts_iterator.HasNext(); sts_iterator.Advance()) {
    const TransportSecurityState::STSState& sts_state =
        sts_iterator.domain_state();

    base::Value serialized(base::Value::Type::DICTIONARY);
    serialized.SetStringKey(
        kHostname, HashedDomainToExternalString(sts_iterator.hostname()));
    serialized.SetBoolKey(kStsIncludeSubdomains, sts_state.include_subdomains);
    serialized.SetDoubleKey(kStsObserved, sts_state.last_observed.ToDoubleT());
    serialized.SetDoubleKey(kExpiry, sts_state.expiry.ToDoubleT());

    switch (sts_state.upgrade_mode) {
      case TransportSecurityState::STSState::MODE_FORCE_HTTPS:
        serialized.SetStringKey(kMode, kForceHTTPS);
        break;
      case TransportSecurityState::STSState::MODE_DEFAULT:
        serialized.SetStringKey(kMode, kDefault);
        break;
    }

    sts_list.Append(std::move(serialized));
  }
  return sts_list;
}

// Deserializes STS data from a Value created by the above method.
void DeserializeSTSData(const base::Value& sts_list,
                        TransportSecurityState* state) {
  if (!sts_list.is_list())
    return;

  base::Time current_time(base::Time::Now());

  for (const base::Value& sts_entry : sts_list.GetList()) {
    if (!sts_entry.is_dict())
      continue;

    const std::string* hostname = sts_entry.FindStringKey(kHostname);
    base::Optional<bool> sts_include_subdomains =
        sts_entry.FindBoolKey(kStsIncludeSubdomains);
    base::Optional<double> sts_observed = sts_entry.FindDoubleKey(kStsObserved);
    base::Optional<double> expiry = sts_entry.FindDoubleKey(kExpiry);
    const std::string* mode = sts_entry.FindStringKey(kMode);

    if (!hostname || !sts_include_subdomains.has_value() ||
        !sts_observed.has_value() || !expiry.has_value() || !mode) {
      continue;
    }

    TransportSecurityState::STSState sts_state;
    sts_state.include_subdomains = *sts_include_subdomains;
    sts_state.last_observed = base::Time::FromDoubleT(*sts_observed);
    sts_state.expiry = base::Time::FromDoubleT(*expiry);

    if (*mode == kForceHTTPS) {
      sts_state.upgrade_mode =
          TransportSecurityState::STSState::MODE_FORCE_HTTPS;
    } else if (*mode == kDefault) {
      sts_state.upgrade_mode = TransportSecurityState::STSState::MODE_DEFAULT;
    } else {
      continue;
    }

    if (sts_state.expiry < current_time || !sts_state.ShouldUpgradeToSSL())
      continue;

    std::string hashed = ExternalStringToHashedDomain(*hostname);
    if (hashed.empty())
      continue;

    state->AddOrUpdateEnabledSTSHosts(hashed, sts_state);
  }
}

// Serializes Expect-CT data from |state| to a Value.
base::Value SerializeExpectCTData(TransportSecurityState* state) {
  base::Value ct_list(base::Value::Type::LIST);

  if (!IsDynamicExpectCTEnabled())
    return ct_list;

  TransportSecurityState::ExpectCTStateIterator expect_ct_iterator(*state);
  for (; expect_ct_iterator.HasNext(); expect_ct_iterator.Advance()) {
    const TransportSecurityState::ExpectCTState& expect_ct_state =
        expect_ct_iterator.domain_state();

    base::DictionaryValue ct_entry;

    base::Value network_isolation_key_value;
    // Don't serialize entries with transient NetworkIsolationKeys.
    if (!expect_ct_iterator.network_isolation_key().ToValue(
            &network_isolation_key_value)) {
      continue;
    }
    ct_entry.SetKey(kNetworkIsolationKey,
                    std::move(network_isolation_key_value));

    ct_entry.SetStringKey(
        kHostname, HashedDomainToExternalString(expect_ct_iterator.hostname()));
    ct_entry.SetDoubleKey(kExpectCTObserved,
                          expect_ct_state.last_observed.ToDoubleT());
    ct_entry.SetDoubleKey(kExpectCTExpiry, expect_ct_state.expiry.ToDoubleT());
    ct_entry.SetBoolKey(kExpectCTEnforce, expect_ct_state.enforce);
    ct_entry.SetStringKey(kExpectCTReportUri,
                          expect_ct_state.report_uri.spec());

    ct_list.Append(std::move(ct_entry));
  }

  return ct_list;
}

// Deserializes Expect-CT data from a Value created by the above method.
void DeserializeExpectCTData(const base::Value& ct_list,
                             TransportSecurityState* state) {
  if (!ct_list.is_list())
    return;
  bool partition_by_nik = base::FeatureList::IsEnabled(
      features::kPartitionExpectCTStateByNetworkIsolationKey);

  const base::Time current_time(base::Time::Now());

  for (const base::Value& ct_entry : ct_list.GetList()) {
    if (!ct_entry.is_dict())
      continue;

    const std::string* hostname = ct_entry.FindStringKey(kHostname);
    const base::Value* network_isolation_key_value =
        ct_entry.FindKey(kNetworkIsolationKey);
    base::Optional<double> expect_ct_last_observed =
        ct_entry.FindDoubleKey(kExpectCTObserved);
    base::Optional<double> expect_ct_expiry =
        ct_entry.FindDoubleKey(kExpectCTExpiry);
    base::Optional<bool> expect_ct_enforce =
        ct_entry.FindBoolKey(kExpectCTEnforce);
    const std::string* expect_ct_report_uri =
        ct_entry.FindStringKey(kExpectCTReportUri);

    if (!hostname || !network_isolation_key_value ||
        !expect_ct_last_observed.has_value() || !expect_ct_expiry.has_value() ||
        !expect_ct_enforce.has_value() || !expect_ct_report_uri) {
      continue;
    }

    TransportSecurityState::ExpectCTState expect_ct_state;
    expect_ct_state.last_observed =
        base::Time::FromDoubleT(*expect_ct_last_observed);
    expect_ct_state.expiry = base::Time::FromDoubleT(*expect_ct_expiry);
    expect_ct_state.enforce = *expect_ct_enforce;

    GURL report_uri(*expect_ct_report_uri);
    if (report_uri.is_valid())
      expect_ct_state.report_uri = report_uri;

    if (expect_ct_state.expiry < current_time ||
        (!expect_ct_state.enforce && expect_ct_state.report_uri.is_empty())) {
      continue;
    }

    std::string hashed = ExternalStringToHashedDomain(*hostname);
    if (hashed.empty())
      continue;

    NetworkIsolationKey network_isolation_key;
    if (!NetworkIsolationKey::FromValue(*network_isolation_key_value,
                                        &network_isolation_key)) {
      continue;
    }

    // If Expect-CT is not being partitioned by NetworkIsolationKey, but
    // |network_isolation_key| is not empty, drop the entry, to avoid ambiguity
    // and favor entries that were saved with an empty NetworkIsolationKey.
    if (!partition_by_nik && !network_isolation_key.IsEmpty())
      continue;

    state->AddOrUpdateEnabledExpectCTHosts(hashed, network_isolation_key,
                                           expect_ct_state);
  }
}

// Handles deserializing |kExpectCTSubdictionary| in dictionaries that use the
// obsolete format. Populates |state| with the values from the Expect-CT
// subdictionary in |parsed|. Returns false if |parsed| is malformed (e.g.
// missing a required Expect-CT key) and true otherwise. Note that true does not
// necessarily mean that Expect-CT state was present in |parsed|.
//
// TODO(mmenke): Remove once the obsolete format is no longer supported.
bool DeserializeObsoleteExpectCTState(
    const base::DictionaryValue* parsed,
    TransportSecurityState::ExpectCTState* state) {
  const base::DictionaryValue* expect_ct_subdictionary;
  if (!parsed->GetDictionary(kExpectCTSubdictionary,
                             &expect_ct_subdictionary)) {
    // Expect-CT data is not required, so this item is not malformed.
    return true;
  }
  double observed;
  bool has_observed =
      expect_ct_subdictionary->GetDouble(kExpectCTObserved, &observed);
  double expiry;
  bool has_expiry =
      expect_ct_subdictionary->GetDouble(kExpectCTExpiry, &expiry);
  bool enforce;
  bool has_enforce =
      expect_ct_subdictionary->GetBoolean(kExpectCTEnforce, &enforce);
  std::string report_uri_str;
  bool has_report_uri =
      expect_ct_subdictionary->GetString(kExpectCTReportUri, &report_uri_str);

  // If an Expect-CT subdictionary is present, it must have the required keys.
  if (!has_observed || !has_expiry || !has_enforce)
    return false;

  state->last_observed = base::Time::FromDoubleT(observed);
  state->expiry = base::Time::FromDoubleT(expiry);
  state->enforce = enforce;
  if (has_report_uri) {
    GURL report_uri(report_uri_str);
    if (report_uri.is_valid())
      state->report_uri = report_uri;
  }
  return true;
}

void OnWriteFinishedTask(scoped_refptr<base::SequencedTaskRunner> task_runner,
                         base::OnceClosure callback,
                         bool result) {
  task_runner->PostTask(FROM_HERE, std::move(callback));
}

}  // namespace

TransportSecurityPersister::TransportSecurityPersister(
    TransportSecurityState* state,
    const base::FilePath& profile_path,
    const scoped_refptr<base::SequencedTaskRunner>& background_runner)
    : transport_security_state_(state),
      writer_(profile_path.AppendASCII("TransportSecurity"), background_runner),
      foreground_runner_(base::ThreadTaskRunnerHandle::Get()),
      background_runner_(background_runner) {
  transport_security_state_->SetDelegate(this);

  base::PostTaskAndReplyWithResult(
      background_runner_.get(), FROM_HERE,
      base::BindOnce(&LoadState, writer_.path()),
      base::BindOnce(&TransportSecurityPersister::CompleteLoad,
                     weak_ptr_factory_.GetWeakPtr()));
}

TransportSecurityPersister::~TransportSecurityPersister() {
  DCHECK(foreground_runner_->RunsTasksInCurrentSequence());

  if (writer_.HasPendingWrite())
    writer_.DoScheduledWrite();

  transport_security_state_->SetDelegate(nullptr);
}

void TransportSecurityPersister::StateIsDirty(TransportSecurityState* state) {
  DCHECK(foreground_runner_->RunsTasksInCurrentSequence());
  DCHECK_EQ(transport_security_state_, state);

  writer_.ScheduleWrite(this);
}

void TransportSecurityPersister::WriteNow(TransportSecurityState* state,
                                          base::OnceClosure callback) {
  DCHECK(foreground_runner_->RunsTasksInCurrentSequence());
  DCHECK_EQ(transport_security_state_, state);

  writer_.RegisterOnNextWriteCallbacks(
      base::OnceClosure(),
      base::BindOnce(
          &OnWriteFinishedTask, foreground_runner_,
          base::BindOnce(&TransportSecurityPersister::OnWriteFinished,
                         weak_ptr_factory_.GetWeakPtr(), std::move(callback))));
  auto data = std::make_unique<std::string>();
  SerializeData(data.get());
  writer_.WriteNow(std::move(data));
}

void TransportSecurityPersister::OnWriteFinished(base::OnceClosure callback) {
  DCHECK(foreground_runner_->RunsTasksInCurrentSequence());
  std::move(callback).Run();
}

bool TransportSecurityPersister::SerializeData(std::string* output) {
  DCHECK(foreground_runner_->RunsTasksInCurrentSequence());

  base::Value toplevel(base::Value::Type::DICTIONARY);
  toplevel.SetIntKey(kVersionKey, kCurrentVersionValue);
  toplevel.SetKey(kSTSKey, SerializeSTSData(transport_security_state_));
  toplevel.SetKey(kExpectCTKey,
                  SerializeExpectCTData(transport_security_state_));

  base::JSONWriter::Write(toplevel, output);
  return true;
}

bool TransportSecurityPersister::LoadEntries(const std::string& serialized,
                                             bool* data_in_old_format) {
  DCHECK(foreground_runner_->RunsTasksInCurrentSequence());

  transport_security_state_->ClearDynamicData();
  return Deserialize(serialized, data_in_old_format, transport_security_state_);
}

bool TransportSecurityPersister::Deserialize(const std::string& serialized,
                                             bool* data_in_old_format,
                                             TransportSecurityState* state) {
  *data_in_old_format = false;
  base::Optional<base::Value> value = base::JSONReader::Read(serialized);
  if (!value || !value->is_dict())
    return false;

  // Old dictionaries don't have a version number, so try the obsolete format if
  // there's no integer version number.
  base::Optional<int> version = value->FindIntKey(kVersionKey);
  if (!version) {
    bool dirty_unused = false;
    bool success = DeserializeObsoleteData(*value, &dirty_unused, state);
    // If successfully loaded data from a file in the old format, need to
    // overwrite the file with the newer format.
    *data_in_old_format = success;
    return success;
  }

  if (*version != kCurrentVersionValue)
    return false;

  base::Value* sts_value = value->FindKey(kSTSKey);
  if (sts_value)
    DeserializeSTSData(*sts_value, state);

  base::Value* expect_ct_value = value->FindKey(kExpectCTKey);
  if (expect_ct_value)
    DeserializeExpectCTData(*expect_ct_value, state);

  UMA_HISTOGRAM_CUSTOM_COUNTS("Net.ExpectCT.EntriesOnLoad",
                              state->num_expect_ct_entries(), 1 /* min */,
                              2000 /* max */, 40 /* buckets */);
  return true;
}

bool TransportSecurityPersister::DeserializeObsoleteData(
    const base::Value& value,
    bool* dirty,
    TransportSecurityState* state) {
  const base::Time current_time(base::Time::Now());
  bool dirtied = false;

  const base::DictionaryValue* dict_value = nullptr;
  int rv = value.GetAsDictionary(&dict_value);
  // The one caller ensures |value| is of Value::Type::DICTIONARY already,
  // though it doesn't extract a DictionaryValue*, since that is the deprecated
  // way to use dictionaries.
  DCHECK(rv);

  for (base::DictionaryValue::Iterator i(*dict_value);
       !i.IsAtEnd(); i.Advance()) {
    const base::DictionaryValue* parsed = nullptr;
    if (!i.value().GetAsDictionary(&parsed)) {
      LOG(WARNING) << "Could not parse entry " << i.key() << "; skipping entry";
      continue;
    }

    TransportSecurityState::STSState sts_state;
    TransportSecurityState::ExpectCTState expect_ct_state;

    // kIncludeSubdomains is a legacy synonym for kStsIncludeSubdomains. Parse
    // at least one of these properties, preferably the new one.
    bool include_subdomains = false;
    bool parsed_include_subdomains = parsed->GetBoolean(kIncludeSubdomains,
                                                        &include_subdomains);
    sts_state.include_subdomains = include_subdomains;
    if (parsed->GetBoolean(kStsIncludeSubdomains, &include_subdomains)) {
      sts_state.include_subdomains = include_subdomains;
      parsed_include_subdomains = true;
    }

    std::string mode_string;
    double expiry = 0;
    if (!parsed_include_subdomains ||
        !parsed->GetString(kMode, &mode_string) ||
        !parsed->GetDouble(kExpiry, &expiry)) {
      LOG(WARNING) << "Could not parse some elements of entry " << i.key()
                   << "; skipping entry";
      continue;
    }

    if (mode_string == kForceHTTPS || mode_string == kStrict) {
      sts_state.upgrade_mode =
          TransportSecurityState::STSState::MODE_FORCE_HTTPS;
    } else if (mode_string == kDefault || mode_string == kPinningOnly) {
      sts_state.upgrade_mode = TransportSecurityState::STSState::MODE_DEFAULT;
    } else {
      LOG(WARNING) << "Unknown TransportSecurityState mode string "
                   << mode_string << " found for entry " << i.key()
                   << "; skipping entry";
      continue;
    }

    sts_state.expiry = base::Time::FromDoubleT(expiry);

    double sts_observed;
    if (parsed->GetDouble(kStsObserved, &sts_observed)) {
      sts_state.last_observed = base::Time::FromDoubleT(sts_observed);
    } else if (parsed->GetDouble(kCreated, &sts_observed)) {
      // kCreated is a legacy synonym for both kStsObserved.
      sts_state.last_observed = base::Time::FromDoubleT(sts_observed);
    } else {
      // We're migrating an old entry with no observation date. Make sure we
      // write the new date back in a reasonable time frame.
      dirtied = true;
      sts_state.last_observed = base::Time::Now();
    }

    if (!DeserializeObsoleteExpectCTState(parsed, &expect_ct_state)) {
      continue;
    }

    bool has_sts =
        sts_state.expiry > current_time && sts_state.ShouldUpgradeToSSL();
    bool has_expect_ct =
        expect_ct_state.expiry > current_time &&
        (expect_ct_state.enforce || !expect_ct_state.report_uri.is_empty());
    if (!has_sts && !has_expect_ct) {
      // Make sure we dirty the state if we drop an entry. The entries can only
      // be dropped when all the STS and Expect-CT states are expired or
      // invalid.
      dirtied = true;
      continue;
    }

    std::string hashed = ExternalStringToHashedDomain(i.key());
    if (hashed.empty()) {
      dirtied = true;
      continue;
    }

    // Until the on-disk storage is split, there will always be 'null' entries.
    // We only register entries that have actual state.
    if (has_sts)
      state->AddOrUpdateEnabledSTSHosts(hashed, sts_state);
    if (has_expect_ct) {
      // Use empty NetworkIsolationKeys for old data.
      state->AddOrUpdateEnabledExpectCTHosts(hashed, NetworkIsolationKey(),
                                             expect_ct_state);
    }
  }

  *dirty = dirtied;
  return true;
}

void TransportSecurityPersister::CompleteLoad(const std::string& state) {
  DCHECK(foreground_runner_->RunsTasksInCurrentSequence());

  if (state.empty())
    return;

  bool data_in_old_format = false;
  if (!LoadEntries(state, &data_in_old_format))
    return;
  if (data_in_old_format)
    StateIsDirty(transport_security_state_);
}

}  // namespace net
