// © 2018 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html #include "unicode/utypes.h" #if !UCONFIG_NO_FORMATTING // Allow implicit conversion from char16_t* to UnicodeString for this file: // Helpful in toString methods and elsewhere. #define UNISTR_FROM_STRING_EXPLICIT #include "numparse_types.h" #include "numparse_affixes.h" #include "numparse_utils.h" #include "number_utils.h" using namespace icu; using namespace icu::numparse; using namespace icu::numparse::impl; using namespace icu::number; using namespace icu::number::impl; namespace { /** * Helper method to return whether the given AffixPatternMatcher equals the given pattern string. * Either both arguments must be null or the pattern string inside the AffixPatternMatcher must equal * the given pattern string. */ static bool matched(const AffixPatternMatcher* affix, const UnicodeString& patternString) { return (affix == nullptr && patternString.isBogus()) || (affix != nullptr && affix->getPattern() == patternString); } /** * Helper method to return the length of the given AffixPatternMatcher. Returns 0 for null. */ static int32_t length(const AffixPatternMatcher* matcher) { return matcher == nullptr ? 0 : matcher->getPattern().length(); } /** * Helper method to return whether (1) both lhs and rhs are null/invalid, or (2) if they are both * valid, whether they are equal according to operator==. Similar to Java Objects.equals() */ static bool equals(const AffixPatternMatcher* lhs, const AffixPatternMatcher* rhs) { if (lhs == nullptr && rhs == nullptr) { return true; } if (lhs == nullptr || rhs == nullptr) { return false; } return *lhs == *rhs; } } AffixPatternMatcherBuilder::AffixPatternMatcherBuilder(const UnicodeString& pattern, AffixTokenMatcherWarehouse& warehouse, IgnorablesMatcher* ignorables) : fMatchersLen(0), fLastTypeOrCp(0), fPattern(pattern), fWarehouse(warehouse), fIgnorables(ignorables) {} void AffixPatternMatcherBuilder::consumeToken(AffixPatternType type, UChar32 cp, UErrorCode& status) { // This is called by AffixUtils.iterateWithConsumer() for each token. // Add an ignorables matcher between tokens except between two literals, and don't put two // ignorables matchers in a row. if (fIgnorables != nullptr && fMatchersLen > 0 && (fLastTypeOrCp < 0 || !fIgnorables->getSet()->contains(fLastTypeOrCp))) { addMatcher(*fIgnorables); } if (type != TYPE_CODEPOINT) { // Case 1: the token is a symbol. switch (type) { case TYPE_MINUS_SIGN: addMatcher(fWarehouse.minusSign()); break; case TYPE_PLUS_SIGN: addMatcher(fWarehouse.plusSign()); break; case TYPE_PERCENT: addMatcher(fWarehouse.percent()); break; case TYPE_PERMILLE: addMatcher(fWarehouse.permille()); break; case TYPE_CURRENCY_SINGLE: case TYPE_CURRENCY_DOUBLE: case TYPE_CURRENCY_TRIPLE: case TYPE_CURRENCY_QUAD: case TYPE_CURRENCY_QUINT: // All currency symbols use the same matcher addMatcher(fWarehouse.currency(status)); break; default: U_ASSERT(FALSE); } } else if (fIgnorables != nullptr && fIgnorables->getSet()->contains(cp)) { // Case 2: the token is an ignorable literal. // No action necessary: the ignorables matcher has already been added. } else { // Case 3: the token is a non-ignorable literal. addMatcher(fWarehouse.nextCodePointMatcher(cp)); } fLastTypeOrCp = type != TYPE_CODEPOINT ? type : cp; } void AffixPatternMatcherBuilder::addMatcher(NumberParseMatcher& matcher) { if (fMatchersLen >= fMatchers.getCapacity()) { fMatchers.resize(fMatchersLen * 2, fMatchersLen); } fMatchers[fMatchersLen++] = &matcher; } AffixPatternMatcher AffixPatternMatcherBuilder::build() { return AffixPatternMatcher(fMatchers, fMatchersLen, fPattern); } CodePointMatcherWarehouse::CodePointMatcherWarehouse() : codePointCount(0), codePointNumBatches(0) {} CodePointMatcherWarehouse::~CodePointMatcherWarehouse() { // Delete the variable number of batches of code point matchers for (int32_t i = 0; i < codePointNumBatches; i++) { delete[] codePointsOverflow[i]; } } CodePointMatcherWarehouse::CodePointMatcherWarehouse(CodePointMatcherWarehouse&& src) U_NOEXCEPT : codePoints(std::move(src.codePoints)), codePointsOverflow(std::move(src.codePointsOverflow)), codePointCount(src.codePointCount), codePointNumBatches(src.codePointNumBatches) {} CodePointMatcherWarehouse& CodePointMatcherWarehouse::operator=(CodePointMatcherWarehouse&& src) U_NOEXCEPT { codePoints = std::move(src.codePoints); codePointsOverflow = std::move(src.codePointsOverflow); codePointCount = src.codePointCount; codePointNumBatches = src.codePointNumBatches; return *this; } NumberParseMatcher& CodePointMatcherWarehouse::nextCodePointMatcher(UChar32 cp) { if (codePointCount < CODE_POINT_STACK_CAPACITY) { return codePoints[codePointCount++] = {cp}; } int32_t totalCapacity = CODE_POINT_STACK_CAPACITY + codePointNumBatches * CODE_POINT_BATCH_SIZE; if (codePointCount >= totalCapacity) { // Need a new batch auto* nextBatch = new CodePointMatcher[CODE_POINT_BATCH_SIZE]; if (codePointNumBatches >= codePointsOverflow.getCapacity()) { // Need more room for storing pointers to batches codePointsOverflow.resize(codePointNumBatches * 2, codePointNumBatches); } codePointsOverflow[codePointNumBatches++] = nextBatch; } return codePointsOverflow[codePointNumBatches - 1][(codePointCount++ - CODE_POINT_STACK_CAPACITY) % CODE_POINT_BATCH_SIZE] = {cp}; } AffixTokenMatcherWarehouse::AffixTokenMatcherWarehouse(const AffixTokenMatcherSetupData* setupData) : fSetupData(setupData) {} NumberParseMatcher& AffixTokenMatcherWarehouse::minusSign() { return fMinusSign = {fSetupData->dfs, true}; } NumberParseMatcher& AffixTokenMatcherWarehouse::plusSign() { return fPlusSign = {fSetupData->dfs, true}; } NumberParseMatcher& AffixTokenMatcherWarehouse::percent() { return fPercent = {fSetupData->dfs}; } NumberParseMatcher& AffixTokenMatcherWarehouse::permille() { return fPermille = {fSetupData->dfs}; } NumberParseMatcher& AffixTokenMatcherWarehouse::currency(UErrorCode& status) { return fCurrency = {fSetupData->currencySymbols, fSetupData->dfs, fSetupData->parseFlags, status}; } IgnorablesMatcher& AffixTokenMatcherWarehouse::ignorables() { return fSetupData->ignorables; } NumberParseMatcher& AffixTokenMatcherWarehouse::nextCodePointMatcher(UChar32 cp) { return fCodePoints.nextCodePointMatcher(cp); } CodePointMatcher::CodePointMatcher(UChar32 cp) : fCp(cp) {} bool CodePointMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode&) const { if (segment.startsWith(fCp)) { segment.adjustOffsetByCodePoint(); result.setCharsConsumed(segment); } return false; } bool CodePointMatcher::smokeTest(const StringSegment& segment) const { return segment.startsWith(fCp); } UnicodeString CodePointMatcher::toString() const { return u"<CodePoint>"; } AffixPatternMatcher AffixPatternMatcher::fromAffixPattern(const UnicodeString& affixPattern, AffixTokenMatcherWarehouse& tokenWarehouse, parse_flags_t parseFlags, bool* success, UErrorCode& status) { if (affixPattern.isEmpty()) { *success = false; return {}; } *success = true; IgnorablesMatcher* ignorables; if (0 != (parseFlags & PARSE_FLAG_EXACT_AFFIX)) { ignorables = nullptr; } else { ignorables = &tokenWarehouse.ignorables(); } AffixPatternMatcherBuilder builder(affixPattern, tokenWarehouse, ignorables); AffixUtils::iterateWithConsumer(affixPattern, builder, status); return builder.build(); } AffixPatternMatcher::AffixPatternMatcher(MatcherArray& matchers, int32_t matchersLen, const UnicodeString& pattern) : ArraySeriesMatcher(matchers, matchersLen), fPattern(pattern) {} UnicodeString AffixPatternMatcher::getPattern() const { return fPattern.toAliasedUnicodeString(); } bool AffixPatternMatcher::operator==(const AffixPatternMatcher& other) const { return fPattern == other.fPattern; } AffixMatcherWarehouse::AffixMatcherWarehouse(AffixTokenMatcherWarehouse* tokenWarehouse) : fTokenWarehouse(tokenWarehouse) { } bool AffixMatcherWarehouse::isInteresting(const AffixPatternProvider& patternInfo, const IgnorablesMatcher& ignorables, parse_flags_t parseFlags, UErrorCode& status) { UnicodeString posPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_PREFIX); UnicodeString posSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_SUFFIX); UnicodeString negPrefixString; UnicodeString negSuffixString; if (patternInfo.hasNegativeSubpattern()) { negPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_PREFIX); negSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_SUFFIX); } if (0 == (parseFlags & PARSE_FLAG_USE_FULL_AFFIXES) && AffixUtils::containsOnlySymbolsAndIgnorables(posPrefixString, *ignorables.getSet(), status) && AffixUtils::containsOnlySymbolsAndIgnorables(posSuffixString, *ignorables.getSet(), status) && AffixUtils::containsOnlySymbolsAndIgnorables(negPrefixString, *ignorables.getSet(), status) && AffixUtils::containsOnlySymbolsAndIgnorables(negSuffixString, *ignorables.getSet(), status) // HACK: Plus and minus sign are a special case: we accept them trailing only if they are // trailing in the pattern string. && !AffixUtils::containsType(posSuffixString, TYPE_PLUS_SIGN, status) && !AffixUtils::containsType(posSuffixString, TYPE_MINUS_SIGN, status) && !AffixUtils::containsType(negSuffixString, TYPE_PLUS_SIGN, status) && !AffixUtils::containsType(negSuffixString, TYPE_MINUS_SIGN, status)) { // The affixes contain only symbols and ignorables. // No need to generate affix matchers. return false; } return true; } void AffixMatcherWarehouse::createAffixMatchers(const AffixPatternProvider& patternInfo, MutableMatcherCollection& output, const IgnorablesMatcher& ignorables, parse_flags_t parseFlags, UErrorCode& status) { if (!isInteresting(patternInfo, ignorables, parseFlags, status)) { return; } // The affixes have interesting characters, or we are in strict mode. // Use initial capacity of 6, the highest possible number of AffixMatchers. UnicodeString sb; bool includeUnpaired = 0 != (parseFlags & PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES); UNumberSignDisplay signDisplay = (0 != (parseFlags & PARSE_FLAG_PLUS_SIGN_ALLOWED)) ? UNUM_SIGN_ALWAYS : UNUM_SIGN_AUTO; int32_t numAffixMatchers = 0; int32_t numAffixPatternMatchers = 0; AffixPatternMatcher* posPrefix = nullptr; AffixPatternMatcher* posSuffix = nullptr; // Pre-process the affix strings to resolve LDML rules like sign display. for (int8_t signum = 1; signum >= -1; signum--) { // Generate Prefix bool hasPrefix = false; PatternStringUtils::patternInfoToStringBuilder( patternInfo, true, signum, signDisplay, StandardPlural::OTHER, false, sb); fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern( sb, *fTokenWarehouse, parseFlags, &hasPrefix, status); AffixPatternMatcher* prefix = hasPrefix ? &fAffixPatternMatchers[numAffixPatternMatchers++] : nullptr; // Generate Suffix bool hasSuffix = false; PatternStringUtils::patternInfoToStringBuilder( patternInfo, false, signum, signDisplay, StandardPlural::OTHER, false, sb); fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern( sb, *fTokenWarehouse, parseFlags, &hasSuffix, status); AffixPatternMatcher* suffix = hasSuffix ? &fAffixPatternMatchers[numAffixPatternMatchers++] : nullptr; if (signum == 1) { posPrefix = prefix; posSuffix = suffix; } else if (equals(prefix, posPrefix) && equals(suffix, posSuffix)) { // Skip adding these matchers (we already have equivalents) continue; } // Flags for setting in the ParsedNumber; the token matchers may add more. int flags = (signum == -1) ? FLAG_NEGATIVE : 0; // Note: it is indeed possible for posPrefix and posSuffix to both be null. // We still need to add that matcher for strict mode to work. fAffixMatchers[numAffixMatchers++] = {prefix, suffix, flags}; if (includeUnpaired && prefix != nullptr && suffix != nullptr) { // The following if statements are designed to prevent adding two identical matchers. if (signum == 1 || !equals(prefix, posPrefix)) { fAffixMatchers[numAffixMatchers++] = {prefix, nullptr, flags}; } if (signum == 1 || !equals(suffix, posSuffix)) { fAffixMatchers[numAffixMatchers++] = {nullptr, suffix, flags}; } } } // Put the AffixMatchers in order, and then add them to the output. // Since there are at most 9 elements, do a simple-to-implement bubble sort. bool madeChanges; do { madeChanges = false; for (int32_t i = 1; i < numAffixMatchers; i++) { if (fAffixMatchers[i - 1].compareTo(fAffixMatchers[i]) > 0) { madeChanges = true; AffixMatcher temp = std::move(fAffixMatchers[i - 1]); fAffixMatchers[i - 1] = std::move(fAffixMatchers[i]); fAffixMatchers[i] = std::move(temp); } } } while (madeChanges); for (int32_t i = 0; i < numAffixMatchers; i++) { // Enable the following line to debug affixes //std::cout << "Adding affix matcher: " << CStr(fAffixMatchers[i].toString())() << std::endl; output.addMatcher(fAffixMatchers[i]); } } AffixMatcher::AffixMatcher(AffixPatternMatcher* prefix, AffixPatternMatcher* suffix, result_flags_t flags) : fPrefix(prefix), fSuffix(suffix), fFlags(flags) {} bool AffixMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const { if (!result.seenNumber()) { // Prefix // Do not match if: // 1. We have already seen a prefix (result.prefix != null) // 2. The prefix in this AffixMatcher is empty (prefix == null) if (!result.prefix.isBogus() || fPrefix == nullptr) { return false; } // Attempt to match the prefix. int initialOffset = segment.getOffset(); bool maybeMore = fPrefix->match(segment, result, status); if (initialOffset != segment.getOffset()) { result.prefix = fPrefix->getPattern(); } return maybeMore; } else { // Suffix // Do not match if: // 1. We have already seen a suffix (result.suffix != null) // 2. The suffix in this AffixMatcher is empty (suffix == null) // 3. The matched prefix does not equal this AffixMatcher's prefix if (!result.suffix.isBogus() || fSuffix == nullptr || !matched(fPrefix, result.prefix)) { return false; } // Attempt to match the suffix. int initialOffset = segment.getOffset(); bool maybeMore = fSuffix->match(segment, result, status); if (initialOffset != segment.getOffset()) { result.suffix = fSuffix->getPattern(); } return maybeMore; } } bool AffixMatcher::smokeTest(const StringSegment& segment) const { return (fPrefix != nullptr && fPrefix->smokeTest(segment)) || (fSuffix != nullptr && fSuffix->smokeTest(segment)); } void AffixMatcher::postProcess(ParsedNumber& result) const { // Check to see if our affix is the one that was matched. If so, set the flags in the result. if (matched(fPrefix, result.prefix) && matched(fSuffix, result.suffix)) { // Fill in the result prefix and suffix with non-null values (empty string). // Used by strict mode to determine whether an entire affix pair was matched. if (result.prefix.isBogus()) { result.prefix = UnicodeString(); } if (result.suffix.isBogus()) { result.suffix = UnicodeString(); } result.flags |= fFlags; if (fPrefix != nullptr) { fPrefix->postProcess(result); } if (fSuffix != nullptr) { fSuffix->postProcess(result); } } } int8_t AffixMatcher::compareTo(const AffixMatcher& rhs) const { const AffixMatcher& lhs = *this; if (length(lhs.fPrefix) != length(rhs.fPrefix)) { return length(lhs.fPrefix) > length(rhs.fPrefix) ? -1 : 1; } else if (length(lhs.fSuffix) != length(rhs.fSuffix)) { return length(lhs.fSuffix) > length(rhs.fSuffix) ? -1 : 1; } else { return 0; } } UnicodeString AffixMatcher::toString() const { bool isNegative = 0 != (fFlags & FLAG_NEGATIVE); return UnicodeString(u"<Affix") + (isNegative ? u":negative " : u" ") + (fPrefix ? fPrefix->getPattern() : u"null") + u"#" + (fSuffix ? fSuffix->getPattern() : u"null") + u">"; } #endif /* #if !UCONFIG_NO_FORMATTING */