/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "utils/tokenizer.h"
#include <algorithm>
#include "utils/base/logging.h"
#include "utils/base/macros.h"
#include "utils/strings/utf8.h"
namespace libtextclassifier3 {
Tokenizer::Tokenizer(
const TokenizationType type, const UniLib* unilib,
const std::vector<const TokenizationCodepointRange*>& codepoint_ranges,
const std::vector<const CodepointRange*>&
internal_tokenizer_codepoint_ranges,
const bool split_on_script_change,
const bool icu_preserve_whitespace_tokens)
: type_(type),
unilib_(unilib),
split_on_script_change_(split_on_script_change),
icu_preserve_whitespace_tokens_(icu_preserve_whitespace_tokens) {
for (const TokenizationCodepointRange* range : codepoint_ranges) {
codepoint_ranges_.emplace_back(range->UnPack());
}
std::sort(codepoint_ranges_.begin(), codepoint_ranges_.end(),
[](const std::unique_ptr<const TokenizationCodepointRangeT>& a,
const std::unique_ptr<const TokenizationCodepointRangeT>& b) {
return a->start < b->start;
});
SortCodepointRanges(internal_tokenizer_codepoint_ranges,
&internal_tokenizer_codepoint_ranges_);
}
const TokenizationCodepointRangeT* Tokenizer::FindTokenizationRange(
int codepoint) const {
auto it = std::lower_bound(
codepoint_ranges_.begin(), codepoint_ranges_.end(), codepoint,
[](const std::unique_ptr<const TokenizationCodepointRangeT>& range,
int codepoint) {
// This function compares range with the codepoint for the purpose of
// finding the first greater or equal range. Because of the use of
// std::lower_bound it needs to return true when range < codepoint;
// the first time it will return false the lower bound is found and
// returned.
//
// It might seem weird that the condition is range.end <= codepoint
// here but when codepoint == range.end it means it's actually just
// outside of the range, thus the range is less than the codepoint.
return range->end <= codepoint;
});
if (it != codepoint_ranges_.end() && (*it)->start <= codepoint &&
(*it)->end > codepoint) {
return it->get();
} else {
return nullptr;
}
}
void Tokenizer::GetScriptAndRole(char32 codepoint,
TokenizationCodepointRange_::Role* role,
int* script) const {
const TokenizationCodepointRangeT* range = FindTokenizationRange(codepoint);
if (range) {
*role = range->role;
*script = range->script_id;
} else {
*role = TokenizationCodepointRange_::Role_DEFAULT_ROLE;
*script = kUnknownScript;
}
}
std::vector<Token> Tokenizer::Tokenize(const std::string& text) const {
UnicodeText text_unicode = UTF8ToUnicodeText(text, /*do_copy=*/false);
return Tokenize(text_unicode);
}
std::vector<Token> Tokenizer::Tokenize(const UnicodeText& text_unicode) const {
switch (type_) {
case TokenizationType_INTERNAL_TOKENIZER:
return InternalTokenize(text_unicode);
case TokenizationType_ICU:
TC3_FALLTHROUGH_INTENDED;
case TokenizationType_MIXED: {
std::vector<Token> result;
if (!ICUTokenize(text_unicode, &result)) {
return {};
}
if (type_ == TokenizationType_MIXED) {
InternalRetokenize(text_unicode, &result);
}
return result;
}
default:
TC3_LOG(ERROR) << "Unknown tokenization type specified. Using internal.";
return InternalTokenize(text_unicode);
}
}
std::vector<Token> Tokenizer::InternalTokenize(
const UnicodeText& text_unicode) const {
std::vector<Token> result;
Token new_token("", 0, 0);
int codepoint_index = 0;
int last_script = kInvalidScript;
for (auto it = text_unicode.begin(); it != text_unicode.end();
++it, ++codepoint_index) {
TokenizationCodepointRange_::Role role;
int script;
GetScriptAndRole(*it, &role, &script);
if (role & TokenizationCodepointRange_::Role_SPLIT_BEFORE ||
(split_on_script_change_ && last_script != kInvalidScript &&
last_script != script)) {
if (!new_token.value.empty()) {
result.push_back(new_token);
}
new_token = Token("", codepoint_index, codepoint_index);
}
if (!(role & TokenizationCodepointRange_::Role_DISCARD_CODEPOINT)) {
new_token.value += std::string(
it.utf8_data(),
it.utf8_data() + GetNumBytesForNonZeroUTF8Char(it.utf8_data()));
++new_token.end;
}
if (role & TokenizationCodepointRange_::Role_SPLIT_AFTER) {
if (!new_token.value.empty()) {
result.push_back(new_token);
}
new_token = Token("", codepoint_index + 1, codepoint_index + 1);
}
last_script = script;
}
if (!new_token.value.empty()) {
result.push_back(new_token);
}
return result;
}
void Tokenizer::TokenizeSubstring(const UnicodeText& unicode_text,
CodepointSpan span,
std::vector<Token>* result) const {
if (span.first < 0) {
// There is no span to tokenize.
return;
}
// Extract the substring.
UnicodeText text = UnicodeText::Substring(unicode_text, span.first,
span.second, /*do_copy=*/false);
// Run the tokenizer and update the token bounds to reflect the offset of the
// substring.
std::vector<Token> tokens = InternalTokenize(text);
// Avoids progressive capacity increases in the for loop.
result->reserve(result->size() + tokens.size());
for (Token& token : tokens) {
token.start += span.first;
token.end += span.first;
result->emplace_back(std::move(token));
}
}
void Tokenizer::InternalRetokenize(const UnicodeText& unicode_text,
std::vector<Token>* tokens) const {
std::vector<Token> result;
CodepointSpan span(-1, -1);
for (Token& token : *tokens) {
const UnicodeText unicode_token_value =
UTF8ToUnicodeText(token.value, /*do_copy=*/false);
bool should_retokenize = true;
for (const int codepoint : unicode_token_value) {
if (!IsCodepointInRanges(codepoint,
internal_tokenizer_codepoint_ranges_)) {
should_retokenize = false;
break;
}
}
if (should_retokenize) {
if (span.first < 0) {
span.first = token.start;
}
span.second = token.end;
} else {
TokenizeSubstring(unicode_text, span, &result);
span.first = -1;
result.emplace_back(std::move(token));
}
}
TokenizeSubstring(unicode_text, span, &result);
*tokens = std::move(result);
}
bool Tokenizer::ICUTokenize(const UnicodeText& context_unicode,
std::vector<Token>* result) const {
std::unique_ptr<UniLib::BreakIterator> break_iterator =
unilib_->CreateBreakIterator(context_unicode);
if (!break_iterator) {
return false;
}
int last_break_index = 0;
int break_index = 0;
int last_unicode_index = 0;
int unicode_index = 0;
auto token_begin_it = context_unicode.begin();
while ((break_index = break_iterator->Next()) !=
UniLib::BreakIterator::kDone) {
const int token_length = break_index - last_break_index;
unicode_index = last_unicode_index + token_length;
auto token_end_it = token_begin_it;
std::advance(token_end_it, token_length);
// Determine if the whole token is whitespace.
bool is_whitespace = true;
for (auto char_it = token_begin_it; char_it < token_end_it; ++char_it) {
if (!unilib_->IsWhitespace(*char_it)) {
is_whitespace = false;
break;
}
}
const std::string token =
context_unicode.UTF8Substring(token_begin_it, token_end_it);
if (!is_whitespace || icu_preserve_whitespace_tokens_) {
result->push_back(Token(token, last_unicode_index, unicode_index));
}
last_break_index = break_index;
last_unicode_index = unicode_index;
token_begin_it = token_end_it;
}
return true;
}
} // namespace libtextclassifier3