mirror of
https://github.com/facebook/react-native.git
synced 2025-11-01 09:14:26 +00:00
Add standards based lexer/tokenizer for CSS values (#42626)
Summary: Pull Request resolved: https://github.com/facebook/react-native/pull/42626 Right now this is done by some combination of string splitting, JS side regexing, etc. This will not scale for math expressions, and gets more and more annoying when we try to add more. In preparation for adding more units, this adds a tokenizer/lexer for a subset of CSS grammar, based on the spec. This is not hooked up to anything yet, and doesn't add a parser. The algorithm uses a subset of the comprehensive instructions provided at https://www.w3.org/TR/css-syntax-3/#tokenizer-algorithms as a reference, with some major simplifications. Changelog: [Internal] Reviewed By: joevilches Differential Revision: D53030661 fbshipit-source-id: c48bc572c5e02daee0b05e91830f2441528193d1
This commit is contained in:
Nick Gerleman
committed by
Facebook GitHub Bot
Facebook GitHub Bot
parent
359738b6ca
commit
bc6105016c
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
|
||||
#include <react/debug/react_native_assert.h>
|
||||
#include <react/renderer/components/view/CSSTokenizer.h>
|
||||
|
||||
namespace facebook::react {
|
||||
|
||||
CSSTokenizer::CSSTokenizer(std::string_view characters)
|
||||
: remainingCharacters_{characters} {}
|
||||
|
||||
CSSToken CSSTokenizer::next() {
|
||||
// https://www.w3.org/TR/css-syntax-3/#token-diagrams
|
||||
char nextChar = peek();
|
||||
if (isWhitespace(nextChar)) {
|
||||
return consumeWhitespace();
|
||||
} else if (nextChar == '+') {
|
||||
if (isDigit(peekNext())) {
|
||||
return consumeNumeric();
|
||||
} else {
|
||||
return consumeDelim();
|
||||
}
|
||||
} else if (nextChar == '-') {
|
||||
if (isDigit(peekNext())) {
|
||||
return consumeNumeric();
|
||||
} else {
|
||||
return consumeDelim();
|
||||
}
|
||||
} else if (isDigit(nextChar)) {
|
||||
return consumeNumeric();
|
||||
} else if (isIdentStart(nextChar)) {
|
||||
return consumeIdent();
|
||||
} else if (nextChar == '\0') {
|
||||
return CSSToken{CSSTokenType::EndOfFile};
|
||||
} else {
|
||||
return consumeDelim();
|
||||
}
|
||||
}
|
||||
|
||||
char CSSTokenizer::peek() const {
|
||||
auto index = position_;
|
||||
return index >= remainingCharacters_.size() ? '\0'
|
||||
: remainingCharacters_[index];
|
||||
}
|
||||
|
||||
char CSSTokenizer::peekNext() const {
|
||||
auto index = position_ + 1;
|
||||
return index >= remainingCharacters_.size() ? '\0'
|
||||
: remainingCharacters_[index];
|
||||
}
|
||||
|
||||
void CSSTokenizer::advance() {
|
||||
react_native_assert(remainingCharacters_.size() > position_);
|
||||
position_ += 1;
|
||||
}
|
||||
|
||||
CSSToken CSSTokenizer::consumeDelim() {
|
||||
advance();
|
||||
return {CSSTokenType::Delim, consumeRunningValue()};
|
||||
}
|
||||
|
||||
CSSToken CSSTokenizer::consumeWhitespace() {
|
||||
while (isWhitespace(peek())) {
|
||||
advance();
|
||||
}
|
||||
|
||||
consumeRunningValue();
|
||||
return CSSToken{CSSTokenType::WhiteSpace};
|
||||
}
|
||||
|
||||
CSSToken CSSTokenizer::consumeNumber() {
|
||||
// https://www.w3.org/TR/css-syntax-3/#consume-number
|
||||
// https://www.w3.org/TR/css-syntax-3/#convert-a-string-to-a-number
|
||||
int32_t signPart = 1.0;
|
||||
if (peek() == '+' || peek() == '-') {
|
||||
if (peek() == '-') {
|
||||
signPart = -1.0;
|
||||
}
|
||||
advance();
|
||||
}
|
||||
|
||||
int32_t intPart = 0;
|
||||
while (isDigit(peek())) {
|
||||
intPart = intPart * 10 + (peek() - '0');
|
||||
advance();
|
||||
}
|
||||
|
||||
int32_t fractionalPart = 0;
|
||||
int32_t fractionDigits = 0;
|
||||
if (peek() == '.') {
|
||||
advance();
|
||||
while (isDigit(peek())) {
|
||||
fractionalPart = fractionalPart * 10 + (peek() - '0');
|
||||
fractionDigits++;
|
||||
advance();
|
||||
}
|
||||
}
|
||||
|
||||
int32_t exponentSign = 1.0;
|
||||
int32_t exponentPart = 0;
|
||||
if (peek() == 'e' || peek() == 'E') {
|
||||
advance();
|
||||
if (peek() == '+' || peek() == '-') {
|
||||
if (peek() == '-') {
|
||||
exponentSign = -1.0;
|
||||
}
|
||||
advance();
|
||||
}
|
||||
|
||||
while (isDigit(peek())) {
|
||||
exponentPart = exponentPart * 10 + (peek() - '0');
|
||||
advance();
|
||||
}
|
||||
}
|
||||
|
||||
auto value = static_cast<float>(
|
||||
signPart * (intPart + (fractionalPart * std::pow(10, -fractionDigits))) *
|
||||
std::pow(10, exponentSign * exponentPart));
|
||||
|
||||
consumeRunningValue();
|
||||
return {CSSTokenType::Number, value};
|
||||
}
|
||||
|
||||
CSSToken CSSTokenizer::consumeNumeric() {
|
||||
// https://www.w3.org/TR/css-syntax-3/#consume-numeric-token
|
||||
auto numberToken = consumeNumber();
|
||||
|
||||
if (isIdent(peek())) {
|
||||
auto ident = consumeIdent();
|
||||
return {
|
||||
CSSTokenType::Dimension,
|
||||
numberToken.numericValue(),
|
||||
ident.stringValue()};
|
||||
} else if (peek() == '%') {
|
||||
advance();
|
||||
consumeRunningValue();
|
||||
return {CSSTokenType::Percent, numberToken.numericValue()};
|
||||
} else {
|
||||
return numberToken;
|
||||
}
|
||||
}
|
||||
|
||||
CSSToken CSSTokenizer::consumeIdent() {
|
||||
// https://www.w3.org/TR/css-syntax-3/#consume-an-ident-sequence
|
||||
while (isIdent(peek())) {
|
||||
advance();
|
||||
}
|
||||
|
||||
return {CSSTokenType::Ident, consumeRunningValue()};
|
||||
}
|
||||
|
||||
std::string_view CSSTokenizer::consumeRunningValue() {
|
||||
auto next = remainingCharacters_.substr(0, position_);
|
||||
remainingCharacters_ = remainingCharacters_.substr(next.size());
|
||||
position_ = 0;
|
||||
return next;
|
||||
}
|
||||
|
||||
/*static*/ bool CSSTokenizer::isDigit(char c) {
|
||||
// https://www.w3.org/TR/css-syntax-3/#digit
|
||||
return c >= '0' && c <= '9';
|
||||
}
|
||||
|
||||
/*static*/ bool CSSTokenizer::isIdentStart(char c) {
|
||||
// https://www.w3.org/TR/css-syntax-3/#ident-start-code-point
|
||||
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' ||
|
||||
static_cast<unsigned char>(c) > 0x80;
|
||||
}
|
||||
|
||||
/*static*/ bool CSSTokenizer::isIdent(char c) {
|
||||
// https://www.w3.org/TR/css-syntax-3/#ident-code-point
|
||||
return isIdentStart(c) || isDigit(c) || c == '-';
|
||||
}
|
||||
|
||||
/*static*/ bool CSSTokenizer::isWhitespace(char c) {
|
||||
// https://www.w3.org/TR/css-syntax-3/#whitespace
|
||||
return c == ' ' || c == '\t' || c == '\r' || c == '\n';
|
||||
}
|
||||
|
||||
} // namespace facebook::react
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string_view>
|
||||
|
||||
namespace facebook::react {
|
||||
|
||||
/**
|
||||
* One of the tokens defined as part of
|
||||
* https://www.w3.org/TR/css-syntax-3/#tokenizer-definitions
|
||||
*/
|
||||
enum class CSSTokenType {
|
||||
Delim,
|
||||
Dimension,
|
||||
EndOfFile,
|
||||
Ident,
|
||||
Number,
|
||||
Percent,
|
||||
WhiteSpace,
|
||||
};
|
||||
|
||||
struct CSSToken {
|
||||
explicit CSSToken(CSSTokenType type) : type_(type) {}
|
||||
CSSToken(CSSTokenType type, std::string_view value)
|
||||
: type_{type}, stringValue_{value} {}
|
||||
CSSToken(CSSTokenType type, float value)
|
||||
: type_{type}, numericValue_{value} {}
|
||||
CSSToken(CSSTokenType type, float value, std::string_view unit)
|
||||
: type_{type}, numericValue_{value}, unit_{unit} {}
|
||||
|
||||
CSSToken(const CSSToken& other) = default;
|
||||
CSSToken(CSSToken&& other) = default;
|
||||
CSSToken& operator=(const CSSToken& other) = default;
|
||||
CSSToken& operator=(CSSToken&& other) = default;
|
||||
|
||||
CSSTokenType type() const {
|
||||
return type_;
|
||||
}
|
||||
|
||||
std::string_view stringValue() const {
|
||||
return stringValue_;
|
||||
}
|
||||
|
||||
float numericValue() const {
|
||||
return numericValue_;
|
||||
}
|
||||
|
||||
std::string_view unit() const {
|
||||
return unit_;
|
||||
}
|
||||
|
||||
bool operator==(const CSSToken& other) const = default;
|
||||
|
||||
private:
|
||||
CSSTokenType type_;
|
||||
std::string_view stringValue_;
|
||||
float numericValue_{0.0f};
|
||||
std::string_view unit_;
|
||||
};
|
||||
|
||||
/**
|
||||
* A minimal tokenizer for a subset of CSS syntax.
|
||||
* `auto`).
|
||||
*
|
||||
* This is based on the W3C CSS Syntax specification, with simplifications made
|
||||
* for syntax which React Native does not attempt to support.
|
||||
* https://www.w3.org/TR/css-syntax-3/#tokenizing-and-parsing
|
||||
*/
|
||||
class CSSTokenizer {
|
||||
public:
|
||||
explicit CSSTokenizer(std::string_view characters);
|
||||
CSSToken next();
|
||||
|
||||
private:
|
||||
char peek() const;
|
||||
char peekNext() const;
|
||||
void advance();
|
||||
|
||||
CSSToken consumeDelim();
|
||||
CSSToken consumeWhitespace();
|
||||
CSSToken consumeNumber();
|
||||
CSSToken consumeNumeric();
|
||||
CSSToken consumeIdent();
|
||||
|
||||
std::string_view consumeRunningValue();
|
||||
|
||||
static bool isDigit(char c);
|
||||
static bool isIdentStart(char c);
|
||||
static bool isIdent(char c);
|
||||
static bool isWhitespace(char c);
|
||||
|
||||
std::string_view remainingCharacters_;
|
||||
size_t position_{0};
|
||||
};
|
||||
} // namespace facebook::react
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <react/renderer/components/view/CSSTokenizer.h>
|
||||
#include <deque>
|
||||
|
||||
namespace facebook::react {
|
||||
|
||||
static void expectTokens(
|
||||
std::string_view characters,
|
||||
std::initializer_list<CSSToken> expectedTokens) {
|
||||
CSSTokenizer tokenizer{characters};
|
||||
|
||||
for (const auto& expectedToken : expectedTokens) {
|
||||
auto nextToken = tokenizer.next();
|
||||
EXPECT_EQ(nextToken.type(), expectedToken.type());
|
||||
EXPECT_EQ(nextToken.stringValue(), expectedToken.stringValue());
|
||||
EXPECT_EQ(nextToken.numericValue(), expectedToken.numericValue());
|
||||
EXPECT_EQ(nextToken.unit(), expectedToken.unit());
|
||||
EXPECT_EQ(nextToken, expectedToken);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(CSSTokenizer, eof_values) {
|
||||
expectTokens("", {CSSToken{CSSTokenType::EndOfFile}});
|
||||
}
|
||||
|
||||
TEST(CSSTokenizer, whitespace_values) {
|
||||
expectTokens(
|
||||
" ",
|
||||
{CSSToken{CSSTokenType::WhiteSpace}, CSSToken{CSSTokenType::EndOfFile}});
|
||||
expectTokens(
|
||||
" \t",
|
||||
{CSSToken{CSSTokenType::WhiteSpace}, CSSToken{CSSTokenType::EndOfFile}});
|
||||
expectTokens(
|
||||
"\n \t",
|
||||
{CSSToken{CSSTokenType::WhiteSpace}, CSSToken{CSSTokenType::EndOfFile}});
|
||||
}
|
||||
|
||||
TEST(CSSTokenizer, ident_values) {
|
||||
expectTokens(
|
||||
"auto",
|
||||
{CSSToken{CSSTokenType::Ident, "auto"},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"inset auto left",
|
||||
{CSSToken{CSSTokenType::Ident, "inset"},
|
||||
CSSToken{CSSTokenType::WhiteSpace},
|
||||
CSSToken{CSSTokenType::Ident, "auto"},
|
||||
CSSToken{CSSTokenType::WhiteSpace},
|
||||
CSSToken{CSSTokenType::Ident, "left"},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
}
|
||||
|
||||
TEST(CSSTokenizer, number_values) {
|
||||
expectTokens(
|
||||
"12",
|
||||
{CSSToken{CSSTokenType::Number, 12.0f},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"-5",
|
||||
{CSSToken{CSSTokenType::Number, -5.0f},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"123.0",
|
||||
{CSSToken{CSSTokenType::Number, 123.0f},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"4.2E-1",
|
||||
{CSSToken{CSSTokenType::Number, 4.2e-1},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"6e-10",
|
||||
{CSSToken{CSSTokenType::Number, 6e-10f},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"+81.07e+0",
|
||||
{CSSToken{CSSTokenType::Number, +81.07e+0},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
}
|
||||
|
||||
TEST(CSSTokenizer, dimension_values) {
|
||||
expectTokens(
|
||||
"12px",
|
||||
{CSSToken{CSSTokenType::Dimension, 12.0f, "px"},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"463.2abc",
|
||||
{CSSToken{CSSTokenType::Dimension, 463.2, "abc"},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
}
|
||||
|
||||
TEST(CSSTokenizer, percent_values) {
|
||||
expectTokens(
|
||||
"12%",
|
||||
{CSSToken{CSSTokenType::Percent, 12.0f},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"-28.5%",
|
||||
{CSSToken{CSSTokenType::Percent, -28.5f},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
}
|
||||
|
||||
TEST(CSSTokenizer, mixed_values) {
|
||||
expectTokens(
|
||||
"12px -100vh",
|
||||
{CSSToken{CSSTokenType::Dimension, 12.0f, "px"},
|
||||
CSSToken{CSSTokenType::WhiteSpace},
|
||||
CSSToken{CSSTokenType::Dimension, -100.0f, "vh"},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
}
|
||||
|
||||
TEST(CSSTokenizer, invalid_values) {
|
||||
expectTokens(
|
||||
"100*",
|
||||
{CSSToken{CSSTokenType::Number, 100.0f},
|
||||
CSSToken{CSSTokenType::Delim, "*"},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"+",
|
||||
{CSSToken{CSSTokenType::Delim, "+"}, CSSToken{CSSTokenType::EndOfFile}});
|
||||
|
||||
expectTokens(
|
||||
"(%",
|
||||
{CSSToken{CSSTokenType::Delim, "("},
|
||||
CSSToken{CSSTokenType::Delim, "%"},
|
||||
CSSToken{CSSTokenType::EndOfFile}});
|
||||
}
|
||||
|
||||
} // namespace facebook::react
|
||||
Reference in New Issue
Block a user