tools/aapt2/compile/Pseudolocalizer.cpp

/*
 * Copyright (C) 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "compile/Pseudolocalizer.h"
#include "util/Util.h"

namespace aapt {

// String basis to generate expansion
static const std::string k_expansion_string = "one two three "
        "four five six seven eight nine ten eleven twelve thirteen "
        "fourteen fiveteen sixteen seventeen nineteen twenty";

// Special unicode characters to override directionality of the words
static const std::string k_rlm = "\u200f";
static const std::string k_rlo = "\u202e";
static const std::string k_pdf = "\u202c";

// Placeholder marks
static const std::string k_placeholder_open = "\u00bb";
static const std::string k_placeholder_close = "\u00ab";

static const char k_arg_start = '{';
static const char k_arg_end = '}';

class PseudoMethodNone : public PseudoMethodImpl {
public:
    std::string text(const StringPiece& text) override { return text.toString(); }
    std::string placeholder(const StringPiece& text) override { return text.toString(); }
};

class PseudoMethodBidi : public PseudoMethodImpl {
public:
    std::string text(const StringPiece& text) override;
    std::string placeholder(const StringPiece& text) override;
};

class PseudoMethodAccent : public PseudoMethodImpl {
public:
    PseudoMethodAccent() : mDepth(0), mWordCount(0), mLength(0) {}
    std::string start() override;
    std::string end() override;
    std::string text(const StringPiece& text) override;
    std::string placeholder(const StringPiece& text) override;
private:
    size_t mDepth;
    size_t mWordCount;
    size_t mLength;
};

Pseudolocalizer::Pseudolocalizer(Method method) : mLastDepth(0) {
    setMethod(method);
}

void Pseudolocalizer::setMethod(Method method) {
    switch (method) {
    case Method::kNone:
        mImpl = util::make_unique<PseudoMethodNone>();
        break;
    case Method::kAccent:
        mImpl = util::make_unique<PseudoMethodAccent>();
        break;
    case Method::kBidi:
        mImpl = util::make_unique<PseudoMethodBidi>();
        break;
    }
}

std::string Pseudolocalizer::text(const StringPiece& text) {
    std::string out;
    size_t depth = mLastDepth;
    size_t lastpos, pos;
    const size_t length = text.size();
    const char* str = text.data();
    bool escaped = false;
    for (lastpos = pos = 0; pos < length; pos++) {
        char16_t c = str[pos];
        if (escaped) {
            escaped = false;
            continue;
        }
        if (c == '\'') {
            escaped = true;
            continue;
        }

        if (c == k_arg_start) {
            depth++;
        } else if (c == k_arg_end && depth) {
            depth--;
        }

        if (mLastDepth != depth || pos == length - 1) {
            bool pseudo = ((mLastDepth % 2) == 0);
            size_t nextpos = pos;
            if (!pseudo || depth == mLastDepth) {
                nextpos++;
            }
            size_t size = nextpos - lastpos;
            if (size) {
                std::string chunk = text.substr(lastpos, size).toString();
                if (pseudo) {
                    chunk = mImpl->text(chunk);
                } else if (str[lastpos] == k_arg_start && str[nextpos - 1] == k_arg_end) {
                    chunk = mImpl->placeholder(chunk);
                }
                out.append(chunk);
            }
            if (pseudo && depth < mLastDepth) { // End of message
                out.append(mImpl->end());
            } else if (!pseudo && depth > mLastDepth) { // Start of message
                out.append(mImpl->start());
            }
            lastpos = nextpos;
            mLastDepth = depth;
        }
    }
    return out;
}

static const char* pseudolocalizeChar(const char c) {
    switch (c) {
        case 'a':   return "\u00e5";
        case 'b':   return "\u0253";
        case 'c':   return "\u00e7";
        case 'd':   return "\u00f0";
        case 'e':   return "\u00e9";
        case 'f':   return "\u0192";
        case 'g':   return "\u011d";
        case 'h':   return "\u0125";
        case 'i':   return "\u00ee";
        case 'j':   return "\u0135";
        case 'k':   return "\u0137";
        case 'l':   return "\u013c";
        case 'm':   return "\u1e3f";
        case 'n':   return "\u00f1";
        case 'o':   return "\u00f6";
        case 'p':   return "\u00fe";
        case 'q':   return "\u0051";
        case 'r':   return "\u0155";
        case 's':   return "\u0161";
        case 't':   return "\u0163";
        case 'u':   return "\u00fb";
        case 'v':   return "\u0056";
        case 'w':   return "\u0175";
        case 'x':   return "\u0445";
        case 'y':   return "\u00fd";
        case 'z':   return "\u017e";
        case 'A':   return "\u00c5";
        case 'B':   return "\u03b2";
        case 'C':   return "\u00c7";
        case 'D':   return "\u00d0";
        case 'E':   return "\u00c9";
        case 'G':   return "\u011c";
        case 'H':   return "\u0124";
        case 'I':   return "\u00ce";
        case 'J':   return "\u0134";
        case 'K':   return "\u0136";
        case 'L':   return "\u013b";
        case 'M':   return "\u1e3e";
        case 'N':   return "\u00d1";
        case 'O':   return "\u00d6";
        case 'P':   return "\u00de";
        case 'Q':   return "\u0071";
        case 'R':   return "\u0154";
        case 'S':   return "\u0160";
        case 'T':   return "\u0162";
        case 'U':   return "\u00db";
        case 'V':   return "\u03bd";
        case 'W':   return "\u0174";
        case 'X':   return "\u00d7";
        case 'Y':   return "\u00dd";
        case 'Z':   return "\u017d";
        case '!':   return "\u00a1";
        case '?':   return "\u00bf";
        case '$':   return "\u20ac";
        default:    return nullptr;
    }
}

static bool isPossibleNormalPlaceholderEnd(const char c) {
    switch (c) {
        case 's': return true;
        case 'S': return true;
        case 'c': return true;
        case 'C': return true;
        case 'd': return true;
        case 'o': return true;
        case 'x': return true;
        case 'X': return true;
        case 'f': return true;
        case 'e': return true;
        case 'E': return true;
        case 'g': return true;
        case 'G': return true;
        case 'a': return true;
        case 'A': return true;
        case 'b': return true;
        case 'B': return true;
        case 'h': return true;
        case 'H': return true;
        case '%': return true;
        case 'n': return true;
        default:  return false;
    }
}

static std::string pseudoGenerateExpansion(const unsigned int length) {
    std::string result = k_expansion_string;
    const char* s = result.data();
    if (result.size() < length) {
        result += " ";
        result += pseudoGenerateExpansion(length - result.size());
    } else {
        int ext = 0;
        // Should contain only whole words, so looking for a space
        for (unsigned int i = length + 1; i < result.size(); ++i) {
            ++ext;
            if (s[i] == ' ') {
                break;
            }
        }
        result = result.substr(0, length + ext);
    }
    return result;
}

std::string PseudoMethodAccent::start() {
    std::string result;
    if (mDepth == 0) {
        result = "[";
    }
    mWordCount = mLength = 0;
    mDepth++;
    return result;
}

std::string PseudoMethodAccent::end() {
    std::string result;
    if (mLength) {
        result += " ";
        result += pseudoGenerateExpansion(mWordCount > 3 ? mLength : mLength / 2);
    }
    mWordCount = mLength = 0;
    mDepth--;
    if (mDepth == 0) {
        result += "]";
    }
    return result;
}

/**
 * Converts characters so they look like they've been localized.
 *
 * Note: This leaves placeholder syntax untouched.
 */
std::string PseudoMethodAccent::text(const StringPiece& source)
{
    const char* s = source.data();
    std::string result;
    const size_t I = source.size();
    bool lastspace = true;
    for (size_t i = 0; i < I; i++) {
        char c = s[i];
        if (c == '%') {
            // Placeholder syntax, no need to pseudolocalize
            std::string chunk;
            bool end = false;
            chunk.append(&c, 1);
            while (!end && i + 1 < I) {
                ++i;
                c = s[i];
                chunk.append(&c, 1);
                if (isPossibleNormalPlaceholderEnd(c)) {
                    end = true;
                } else if (i + 1 < I && c == 't') {
                    ++i;
                    c = s[i];
                    chunk.append(&c, 1);
                    end = true;
                }
            }
            // Treat chunk as a placeholder unless it ends with %.
            result += ((c == '%') ? chunk : placeholder(chunk));
        } else if (c == '<' || c == '&') {
            // html syntax, no need to pseudolocalize
            bool tag_closed = false;
            while (!tag_closed && i < I) {
                if (c == '&') {
                    std::string escapeText;
                    escapeText.append(&c, 1);
                    bool end = false;
                    size_t htmlCodePos = i;
                    while (!end && htmlCodePos < I) {
                        ++htmlCodePos;
                        c = s[htmlCodePos];
                        escapeText.append(&c, 1);
                        // Valid html code
                        if (c == ';') {
                            end = true;
                            i = htmlCodePos;
                        }
                        // Wrong html code
                        else if (!((c == '#' ||
                                 (c >= 'a' && c <= 'z') ||
                                 (c >= 'A' && c <= 'Z') ||
                                 (c >= '0' && c <= '9')))) {
                            end = true;
                        }
                    }
                    result += escapeText;
                    if (escapeText != "&lt;") {
                        tag_closed = true;
                    }
                    continue;
                }
                if (c == '>') {
                    tag_closed = true;
                    result.append(&c, 1);
                    continue;
                }
                result.append(&c, 1);
                i++;
                c = s[i];
            }
        } else {
            // This is a pure text that should be pseudolocalized
            const char* p = pseudolocalizeChar(c);
            if (p != nullptr) {
                result += p;
            } else {
                bool space = isspace(c);
                if (lastspace && !space) {
                    mWordCount++;
                }
                lastspace = space;
                result.append(&c, 1);
            }
            // Count only pseudolocalizable chars and delimiters
            mLength++;
        }
    }
    return result;
}

std::string PseudoMethodAccent::placeholder(const StringPiece& source) {
    // Surround a placeholder with brackets
    return k_placeholder_open + source.toString() + k_placeholder_close;
}

std::string PseudoMethodBidi::text(const StringPiece& source) {
    const char* s = source.data();
    std::string result;
    bool lastspace = true;
    bool space = true;
    for (size_t i = 0; i < source.size(); i++) {
        char c = s[i];
        space = isspace(c);
        if (lastspace && !space) {
            // Word start
            result += k_rlm + k_rlo;
        } else if (!lastspace && space) {
            // Word end
            result += k_pdf + k_rlm;
        }
        lastspace = space;
        result.append(&c, 1);
    }
    if (!lastspace) {
        // End of last word
        result += k_pdf + k_rlm;
    }
    return result;
}

std::string PseudoMethodBidi::placeholder(const StringPiece& source) {
    // Surround a placeholder with directionality change sequence
    return k_rlm + k_rlo + source.toString() + k_pdf + k_rlm;
}

} // namespace aapt