Import latest app updates from streamline

An updated set of source files and initialization was provided by streamline to
address issues observed during initial testing. These files have been updated in
order to generate a new set of app images.
This commit is contained in:
2024-04-11 11:16:51 -07:00
parent cb4c6b4c1a
commit 4a89356390
10435 changed files with 107737 additions and 1220538 deletions
@@ -1,226 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer;
use PhpParser\Error;
use PhpParser\ErrorHandler;
use PhpParser\Lexer;
use PhpParser\Lexer\TokenEmulator\AttributeEmulator;
use PhpParser\Lexer\TokenEmulator\EnumTokenEmulator;
use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
use PhpParser\Lexer\TokenEmulator\ExplicitOctalEmulator;
use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator;
use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
use PhpParser\Lexer\TokenEmulator\MatchTokenEmulator;
use PhpParser\Lexer\TokenEmulator\NullsafeTokenEmulator;
use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
use PhpParser\Lexer\TokenEmulator\ReadonlyFunctionTokenEmulator;
use PhpParser\Lexer\TokenEmulator\ReadonlyTokenEmulator;
use PhpParser\Lexer\TokenEmulator\ReverseEmulator;
use PhpParser\Lexer\TokenEmulator\TokenEmulator;
use PhpParser\PhpVersion;
use PhpParser\Token;
class Emulative extends Lexer {
/** @var array{int, string, string}[] Patches used to reverse changes introduced in the code */
private array $patches = [];
/** @var list<TokenEmulator> */
private array $emulators = [];
private PhpVersion $targetPhpVersion;
private PhpVersion $hostPhpVersion;
/**
* @param PhpVersion|null $phpVersion PHP version to emulate. Defaults to newest supported.
*/
public function __construct(?PhpVersion $phpVersion = null) {
$this->targetPhpVersion = $phpVersion ?? PhpVersion::getNewestSupported();
$this->hostPhpVersion = PhpVersion::getHostVersion();
$emulators = [
new MatchTokenEmulator(),
new NullsafeTokenEmulator(),
new AttributeEmulator(),
new EnumTokenEmulator(),
new ReadonlyTokenEmulator(),
new ExplicitOctalEmulator(),
new ReadonlyFunctionTokenEmulator(),
];
// Collect emulators that are relevant for the PHP version we're running
// and the PHP version we're targeting for emulation.
foreach ($emulators as $emulator) {
$emulatorPhpVersion = $emulator->getPhpVersion();
if ($this->isForwardEmulationNeeded($emulatorPhpVersion)) {
$this->emulators[] = $emulator;
} elseif ($this->isReverseEmulationNeeded($emulatorPhpVersion)) {
$this->emulators[] = new ReverseEmulator($emulator);
}
}
}
public function tokenize(string $code, ?ErrorHandler $errorHandler = null): array {
$emulators = array_filter($this->emulators, function ($emulator) use ($code) {
return $emulator->isEmulationNeeded($code);
});
if (empty($emulators)) {
// Nothing to emulate, yay
return parent::tokenize($code, $errorHandler);
}
if ($errorHandler === null) {
$errorHandler = new ErrorHandler\Throwing();
}
$this->patches = [];
foreach ($emulators as $emulator) {
$code = $emulator->preprocessCode($code, $this->patches);
}
$collector = new ErrorHandler\Collecting();
$tokens = parent::tokenize($code, $collector);
$this->sortPatches();
$tokens = $this->fixupTokens($tokens);
$errors = $collector->getErrors();
if (!empty($errors)) {
$this->fixupErrors($errors);
foreach ($errors as $error) {
$errorHandler->handleError($error);
}
}
foreach ($emulators as $emulator) {
$tokens = $emulator->emulate($code, $tokens);
}
return $tokens;
}
private function isForwardEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
return $this->hostPhpVersion->older($emulatorPhpVersion)
&& $this->targetPhpVersion->newerOrEqual($emulatorPhpVersion);
}
private function isReverseEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
return $this->hostPhpVersion->newerOrEqual($emulatorPhpVersion)
&& $this->targetPhpVersion->older($emulatorPhpVersion);
}
private function sortPatches(): void {
// Patches may be contributed by different emulators.
// Make sure they are sorted by increasing patch position.
usort($this->patches, function ($p1, $p2) {
return $p1[0] <=> $p2[0];
});
}
/**
* @param list<Token> $tokens
* @return list<Token>
*/
private function fixupTokens(array $tokens): array {
if (\count($this->patches) === 0) {
return $tokens;
}
// Load first patch
$patchIdx = 0;
list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
// We use a manual loop over the tokens, because we modify the array on the fly
$posDelta = 0;
$lineDelta = 0;
for ($i = 0, $c = \count($tokens); $i < $c; $i++) {
$token = $tokens[$i];
$pos = $token->pos;
$token->pos += $posDelta;
$token->line += $lineDelta;
$localPosDelta = 0;
$len = \strlen($token->text);
while ($patchPos >= $pos && $patchPos < $pos + $len) {
$patchTextLen = \strlen($patchText);
if ($patchType === 'remove') {
if ($patchPos === $pos && $patchTextLen === $len) {
// Remove token entirely
array_splice($tokens, $i, 1, []);
$i--;
$c--;
} else {
// Remove from token string
$token->text = substr_replace(
$token->text, '', $patchPos - $pos + $localPosDelta, $patchTextLen
);
$localPosDelta -= $patchTextLen;
}
$lineDelta -= \substr_count($patchText, "\n");
} elseif ($patchType === 'add') {
// Insert into the token string
$token->text = substr_replace(
$token->text, $patchText, $patchPos - $pos + $localPosDelta, 0
);
$localPosDelta += $patchTextLen;
$lineDelta += \substr_count($patchText, "\n");
} elseif ($patchType === 'replace') {
// Replace inside the token string
$token->text = substr_replace(
$token->text, $patchText, $patchPos - $pos + $localPosDelta, $patchTextLen
);
} else {
assert(false);
}
// Fetch the next patch
$patchIdx++;
if ($patchIdx >= \count($this->patches)) {
// No more patches. However, we still need to adjust position.
$patchPos = \PHP_INT_MAX;
break;
}
list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
}
$posDelta += $localPosDelta;
}
return $tokens;
}
/**
* Fixup line and position information in errors.
*
* @param Error[] $errors
*/
private function fixupErrors(array $errors): void {
foreach ($errors as $error) {
$attrs = $error->getAttributes();
$posDelta = 0;
$lineDelta = 0;
foreach ($this->patches as $patch) {
list($patchPos, $patchType, $patchText) = $patch;
if ($patchPos >= $attrs['startFilePos']) {
// No longer relevant
break;
}
if ($patchType === 'add') {
$posDelta += strlen($patchText);
$lineDelta += substr_count($patchText, "\n");
} elseif ($patchType === 'remove') {
$posDelta -= strlen($patchText);
$lineDelta -= substr_count($patchText, "\n");
}
}
$attrs['startFilePos'] += $posDelta;
$attrs['endFilePos'] += $posDelta;
$attrs['startLine'] += $lineDelta;
$attrs['endLine'] += $lineDelta;
$error->setAttributes($attrs);
}
}
}
@@ -1,49 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
use PhpParser\Token;
final class AttributeEmulator extends TokenEmulator {
public function getPhpVersion(): PhpVersion {
return PhpVersion::fromComponents(8, 0);
}
public function isEmulationNeeded(string $code): bool {
return strpos($code, '#[') !== false;
}
public function emulate(string $code, array $tokens): array {
// We need to manually iterate and manage a count because we'll change
// the tokens array on the way.
for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
$token = $tokens[$i];
if ($token->text === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1]->text === '[') {
array_splice($tokens, $i, 2, [
new Token(\T_ATTRIBUTE, '#[', $token->line, $token->pos),
]);
$c--;
continue;
}
}
return $tokens;
}
public function reverseEmulate(string $code, array $tokens): array {
// TODO
return $tokens;
}
public function preprocessCode(string $code, array &$patches): string {
$pos = 0;
while (false !== $pos = strpos($code, '#[', $pos)) {
// Replace #[ with %[
$code[$pos] = '%';
$patches[] = [$pos, 'replace', '#'];
$pos += 2;
}
return $code;
}
}
@@ -1,26 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
final class EnumTokenEmulator extends KeywordEmulator {
public function getPhpVersion(): PhpVersion {
return PhpVersion::fromComponents(8, 1);
}
public function getKeywordString(): string {
return 'enum';
}
public function getKeywordToken(): int {
return \T_ENUM;
}
protected function isKeywordContext(array $tokens, int $pos): bool {
return parent::isKeywordContext($tokens, $pos)
&& isset($tokens[$pos + 2])
&& $tokens[$pos + 1]->id === \T_WHITESPACE
&& $tokens[$pos + 2]->id === \T_STRING;
}
}
@@ -1,45 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
use PhpParser\Token;
class ExplicitOctalEmulator extends TokenEmulator {
public function getPhpVersion(): PhpVersion {
return PhpVersion::fromComponents(8, 1);
}
public function isEmulationNeeded(string $code): bool {
return strpos($code, '0o') !== false || strpos($code, '0O') !== false;
}
public function emulate(string $code, array $tokens): array {
for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
$token = $tokens[$i];
if ($token->id == \T_LNUMBER && $token->text === '0' &&
isset($tokens[$i + 1]) && $tokens[$i + 1]->id == \T_STRING &&
preg_match('/[oO][0-7]+(?:_[0-7]+)*/', $tokens[$i + 1]->text)
) {
$tokenKind = $this->resolveIntegerOrFloatToken($tokens[$i + 1]->text);
array_splice($tokens, $i, 2, [
new Token($tokenKind, '0' . $tokens[$i + 1]->text, $token->line, $token->pos),
]);
$c--;
}
}
return $tokens;
}
private function resolveIntegerOrFloatToken(string $str): int {
$str = substr($str, 1);
$str = str_replace('_', '', $str);
$num = octdec($str);
return is_float($num) ? \T_DNUMBER : \T_LNUMBER;
}
public function reverseEmulate(string $code, array $tokens): array {
// Explicit octals were not legal code previously, don't bother.
return $tokens;
}
}
@@ -1,56 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\Token;
abstract class KeywordEmulator extends TokenEmulator {
abstract public function getKeywordString(): string;
abstract public function getKeywordToken(): int;
public function isEmulationNeeded(string $code): bool {
return strpos(strtolower($code), $this->getKeywordString()) !== false;
}
/** @param Token[] $tokens */
protected function isKeywordContext(array $tokens, int $pos): bool {
$previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $pos);
return $previousNonSpaceToken === null || $previousNonSpaceToken->id !== \T_OBJECT_OPERATOR;
}
public function emulate(string $code, array $tokens): array {
$keywordString = $this->getKeywordString();
foreach ($tokens as $i => $token) {
if ($token->id === T_STRING && strtolower($token->text) === $keywordString
&& $this->isKeywordContext($tokens, $i)) {
$token->id = $this->getKeywordToken();
}
}
return $tokens;
}
/** @param Token[] $tokens */
private function getPreviousNonSpaceToken(array $tokens, int $start): ?Token {
for ($i = $start - 1; $i >= 0; --$i) {
if ($tokens[$i]->id === T_WHITESPACE) {
continue;
}
return $tokens[$i];
}
return null;
}
public function reverseEmulate(string $code, array $tokens): array {
$keywordToken = $this->getKeywordToken();
foreach ($tokens as $token) {
if ($token->id === $keywordToken) {
$token->id = \T_STRING;
}
}
return $tokens;
}
}
@@ -1,19 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
final class MatchTokenEmulator extends KeywordEmulator {
public function getPhpVersion(): PhpVersion {
return PhpVersion::fromComponents(8, 0);
}
public function getKeywordString(): string {
return 'match';
}
public function getKeywordToken(): int {
return \T_MATCH;
}
}
@@ -1,60 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
use PhpParser\Token;
final class NullsafeTokenEmulator extends TokenEmulator {
public function getPhpVersion(): PhpVersion {
return PhpVersion::fromComponents(8, 0);
}
public function isEmulationNeeded(string $code): bool {
return strpos($code, '?->') !== false;
}
public function emulate(string $code, array $tokens): array {
// We need to manually iterate and manage a count because we'll change
// the tokens array on the way
for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
$token = $tokens[$i];
if ($token->text === '?' && isset($tokens[$i + 1]) && $tokens[$i + 1]->id === \T_OBJECT_OPERATOR) {
array_splice($tokens, $i, 2, [
new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', $token->line, $token->pos),
]);
$c--;
continue;
}
// Handle ?-> inside encapsed string.
if ($token->id === \T_ENCAPSED_AND_WHITESPACE && isset($tokens[$i - 1])
&& $tokens[$i - 1]->id === \T_VARIABLE
&& preg_match('/^\?->([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)/', $token->text, $matches)
) {
$replacement = [
new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', $token->line, $token->pos),
new Token(\T_STRING, $matches[1], $token->line, $token->pos + 3),
];
$matchLen = \strlen($matches[0]);
if ($matchLen !== \strlen($token->text)) {
$replacement[] = new Token(
\T_ENCAPSED_AND_WHITESPACE,
\substr($token->text, $matchLen),
$token->line, $token->pos + $matchLen
);
}
array_splice($tokens, $i, 1, $replacement);
$c += \count($replacement) - 1;
continue;
}
}
return $tokens;
}
public function reverseEmulate(string $code, array $tokens): array {
// ?-> was not valid code previously, don't bother.
return $tokens;
}
}
@@ -1,31 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
/*
* In PHP 8.1, "readonly(" was special cased in the lexer in order to support functions with
* name readonly. In PHP 8.2, this may conflict with readonly properties having a DNF type. For
* this reason, PHP 8.2 instead treats this as T_READONLY and then handles it specially in the
* parser. This emulator only exists to handle this special case, which is skipped by the
* PHP 8.1 ReadonlyTokenEmulator.
*/
class ReadonlyFunctionTokenEmulator extends KeywordEmulator {
public function getKeywordString(): string {
return 'readonly';
}
public function getKeywordToken(): int {
return \T_READONLY;
}
public function getPhpVersion(): PhpVersion {
return PhpVersion::fromComponents(8, 2);
}
public function reverseEmulate(string $code, array $tokens): array {
// Don't bother
return $tokens;
}
}
@@ -1,31 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
final class ReadonlyTokenEmulator extends KeywordEmulator {
public function getPhpVersion(): PhpVersion {
return PhpVersion::fromComponents(8, 1);
}
public function getKeywordString(): string {
return 'readonly';
}
public function getKeywordToken(): int {
return \T_READONLY;
}
protected function isKeywordContext(array $tokens, int $pos): bool {
if (!parent::isKeywordContext($tokens, $pos)) {
return false;
}
// Support "function readonly("
return !(isset($tokens[$pos + 1]) &&
($tokens[$pos + 1]->text === '(' ||
($tokens[$pos + 1]->id === \T_WHITESPACE &&
isset($tokens[$pos + 2]) &&
$tokens[$pos + 2]->text === '(')));
}
}
@@ -1,37 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
/**
* Reverses emulation direction of the inner emulator.
*/
final class ReverseEmulator extends TokenEmulator {
/** @var TokenEmulator Inner emulator */
private TokenEmulator $emulator;
public function __construct(TokenEmulator $emulator) {
$this->emulator = $emulator;
}
public function getPhpVersion(): PhpVersion {
return $this->emulator->getPhpVersion();
}
public function isEmulationNeeded(string $code): bool {
return $this->emulator->isEmulationNeeded($code);
}
public function emulate(string $code, array $tokens): array {
return $this->emulator->reverseEmulate($code, $tokens);
}
public function reverseEmulate(string $code, array $tokens): array {
return $this->emulator->emulate($code, $tokens);
}
public function preprocessCode(string $code, array &$patches): string {
return $code;
}
}
@@ -1,30 +0,0 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\PhpVersion;
use PhpParser\Token;
/** @internal */
abstract class TokenEmulator {
abstract public function getPhpVersion(): PhpVersion;
abstract public function isEmulationNeeded(string $code): bool;
/**
* @param Token[] $tokens Original tokens
* @return Token[] Modified Tokens
*/
abstract public function emulate(string $code, array $tokens): array;
/**
* @param Token[] $tokens Original tokens
* @return Token[] Modified Tokens
*/
abstract public function reverseEmulate(string $code, array $tokens): array;
/** @param array{int, string, string}[] $patches */
public function preprocessCode(string $code, array &$patches): string {
return $code;
}
}