icu_encoding_detection.cc

// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/i18n/icu_encoding_detection.h"

#include <set>

#include "base/string_util.h"
#include "unicode/ucsdet.h"

namespace base {

bool DetectEncoding(const std::string& text, std::string* encoding) {
  if (IsStringASCII(text)) {
    *encoding = std::string();
    return true;
  }

  UErrorCode status = U_ZERO_ERROR;
  UCharsetDetector* detector = ucsdet_open(&status);
  ucsdet_setText(detector, text.data(), static_cast<int32_t>(text.length()),
                 &status);
  const UCharsetMatch* match = ucsdet_detect(detector, &status);
  if (match == NULL)
    return false;
  const char* detected_encoding = ucsdet_getName(match, &status);
  ucsdet_close(detector);

  if (U_FAILURE(status))
    return false;

  *encoding = detected_encoding;
  return true;
}

bool DetectAllEncodings(const std::string& text,
                        std::vector<std::string>* encodings) {
  UErrorCode status = U_ZERO_ERROR;
  UCharsetDetector* detector = ucsdet_open(&status);
  ucsdet_setText(detector, text.data(), static_cast<int32_t>(text.length()),
                 &status);
  int matches_count = 0;
  const UCharsetMatch** matches = ucsdet_detectAll(detector,
                                                   &matches_count,
                                                   &status);
  if (U_FAILURE(status)) {
    ucsdet_close(detector);
    return false;
  }

  // ICU has some heuristics for encoding detection, such that the more likely
  // encodings should be returned first. However, it doesn't always return
  // all encodings that properly decode |text|, so we'll append more encodings
  // later. To make that efficient, keep track of encodings sniffed in this
  // first phase.
  std::set<std::string> sniffed_encodings;

  encodings->clear();
  for (int i = 0; i < matches_count; i++) {
    UErrorCode get_name_status = U_ZERO_ERROR;
    const char* encoding_name = ucsdet_getName(matches[i], &get_name_status);

    // If we failed to get the encoding's name, ignore the error.
    if (U_FAILURE(get_name_status))
      continue;

    int32_t confidence = ucsdet_getConfidence(matches[i], &get_name_status);

    // We also treat this error as non-fatal.
    if (U_FAILURE(get_name_status))
      continue;

    // A confidence level >= 10 means that the encoding is expected to properly
    // decode the text. Drop all encodings with lower confidence level.
    if (confidence < 10)
      continue;

    encodings->push_back(encoding_name);
    sniffed_encodings.insert(encoding_name);
  }

  // Append all encodings not included earlier, in arbitrary order.
  // TODO(jshin): This shouldn't be necessary, possible ICU bug.
  // See also http://crbug.com/65917.
  UEnumeration* detectable_encodings = ucsdet_getAllDetectableCharsets(detector,
                                                                       &status);
  int detectable_count = uenum_count(detectable_encodings, &status);
  for (int i = 0; i < detectable_count; i++) {
    int name_length;
    const char* name_raw = uenum_next(detectable_encodings,
                                      &name_length,
                                      &status);
    std::string name(name_raw, name_length);
    if (sniffed_encodings.find(name) == sniffed_encodings.end())
      encodings->push_back(name);
  }
  uenum_close(detectable_encodings);

  ucsdet_close(detector);
  return !encodings->empty();
}

}  // namespace base