updated streamline-setup v2

This commit is contained in:
2025-01-15 08:53:49 -08:00
committed by alec.turner
parent a2ce9248f0
commit 4b569f81b0
20228 changed files with 2932048 additions and 63204 deletions
@@ -0,0 +1,412 @@
<?php
declare(strict_types=1);
namespace PhpMyAdmin\SqlParser\Tools;
use PhpMyAdmin\SqlParser\Token;
use function array_filter;
use function array_map;
use function array_merge;
use function array_slice;
use function basename;
use function count;
use function dirname;
use function file;
use function file_put_contents;
use function implode;
use function ksort;
use function preg_match;
use function scandir;
use function sort;
use function sprintf;
use function str_replace;
use function str_split;
use function strlen;
use function strstr;
use function strtoupper;
use function substr;
use function trim;
use const ARRAY_FILTER_USE_KEY;
use const SORT_STRING;
/**
* Used for context generation.
*/
class ContextGenerator
{
/**
* Labels and flags that may be used when defining keywords.
*
* @var array<string, int>
*/
public static $LABELS_FLAGS = [
'(R)' => Token::FLAG_KEYWORD_RESERVED,
'(D)' => Token::FLAG_KEYWORD_DATA_TYPE,
'(K)' => Token::FLAG_KEYWORD_KEY,
'(F)' => Token::FLAG_KEYWORD_FUNCTION,
];
/**
* Documentation links for each context.
*
* @var array<string, string>
*/
public static $LINKS = [
'MySql50000' => 'https://dev.mysql.com/doc/refman/5.0/en/keywords.html',
'MySql50100' => 'https://dev.mysql.com/doc/refman/5.1/en/keywords.html',
'MySql50500' => 'https://dev.mysql.com/doc/refman/5.5/en/keywords.html',
'MySql50600' => 'https://dev.mysql.com/doc/refman/5.6/en/keywords.html',
'MySql50700' => 'https://dev.mysql.com/doc/refman/5.7/en/keywords.html',
'MySql80000' => 'https://dev.mysql.com/doc/refman/8.0/en/keywords.html',
'MySql80100' => 'https://dev.mysql.com/doc/refman/8.1/en/keywords.html',
'MySql80200' => 'https://dev.mysql.com/doc/refman/8.2/en/keywords.html',
'MySql80300' => 'https://dev.mysql.com/doc/refman/8.3/en/keywords.html',
'MySql80400' => 'https://dev.mysql.com/doc/refman/8.4/en/keywords.html',
'MySql90000' => 'https://dev.mysql.com/doc/refman/9.0/en/keywords.html',
'MySql90100' => 'https://dev.mysql.com/doc/refman/9.1/en/keywords.html',
'MariaDb100000' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100100' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100200' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100300' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100400' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100500' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100600' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100700' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100800' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb100900' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb101000' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb101100' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb110000' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb110100' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb110200' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb110300' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb110400' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb110500' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb110600' => 'https://mariadb.com/kb/en/reserved-words/',
'MariaDb110700' => 'https://mariadb.com/kb/en/reserved-words/',
];
/**
* Reversed const <=> int from {@see Token} class to write the constant name instead of its value.
*
* @var array<int, string>
*/
private static $typesNumToConst = [
1 => 'Token::FLAG_KEYWORD',
2 => 'Token::FLAG_KEYWORD_RESERVED',
4 => 'Token::FLAG_KEYWORD_COMPOSED',
8 => 'Token::FLAG_KEYWORD_DATA_TYPE',
16 => 'Token::FLAG_KEYWORD_KEY',
32 => 'Token::FLAG_KEYWORD_FUNCTION',
];
/**
* The template of a context.
*
* Parameters:
* 1 - name
* 2 - class
* 3 - link
* 4 - keywords array
*/
public const TEMPLATE = <<<'PHP'
<?php
declare(strict_types=1);
namespace PhpMyAdmin\SqlParser\Contexts;
use PhpMyAdmin\SqlParser\Context;
use PhpMyAdmin\SqlParser\Token;
/**
* Context for %1$s.
*
* This class was auto-generated from tools/contexts/*.txt.
* Use tools/run_generators.sh for update.
*
* @see %3$s
*/
class %2$s extends Context
{
/**
* List of keywords.
*
* The value associated to each keyword represents its flags.
*
* @see Token
*
* @var array<string,int>
* @phpstan-var non-empty-array<non-empty-string,Token::FLAG_KEYWORD_*|int>
*/
public static $KEYWORDS = [
%4$s ];
}
PHP;
/**
* Sorts an array of words.
*
* @param array<int, list<string>> $arr
*
* @return array<int, list<string>>
*/
public static function sortWords(array &$arr)
{
ksort($arr);
foreach ($arr as &$words) {
sort($words, SORT_STRING);
}
return $arr;
}
/**
* Reads a list of words and sorts it by type, length and keyword.
*
* @param list<string> $files
*
* @return array<int, list<string>>
*/
public static function readWords(array $files)
{
/** @psalm-var list<string> $words */
$words = [];
foreach ($files as $file) {
$words = array_merge($words, file($file));
}
/** @var array<string, int> $types */
$types = [];
for ($i = 0, $count = count($words); $i !== $count; ++$i) {
$value = trim($words[$i]);
if ($value === '') {
continue;
}
$type = Token::FLAG_KEYWORD;
// Reserved, data types, keys, functions, etc. keywords.
foreach (static::$LABELS_FLAGS as $label => $flags) {
if (strstr($value, $label) === false) {
continue;
}
$type |= $flags;
$value = trim(str_replace($label, '', $value));
}
// Composed keyword.
if (strstr($value, ' ') !== false) {
$type |= Token::FLAG_KEYWORD_RESERVED;
$type |= Token::FLAG_KEYWORD_COMPOSED;
}
$value = strtoupper($value);
if (! isset($types[$value])) {
$types[$value] = $type;
} else {
$types[$value] |= $type;
}
}
// Prepare an array in a way to sort by type, then by word.
$ret = [];
foreach ($types as $word => $type) {
$ret[$type][] = $word;
}
return static::sortWords($ret);
}
/**
* Prints an array of a words in PHP format.
*
* @param array<int, list<string>> $words the list of words to be formatted
*/
public static function printWords(array $words): string
{
$ret = '';
foreach ($words as $type => $wordsByType) {
foreach ($wordsByType as $word) {
$ret .= sprintf(" '%s' => %s,\n", $word, self::translateIntTypeToTextConstant($type));
}
}
return $ret;
}
private static function translateIntTypeToTextConstant(int $type): string
{
$matchingFlags = array_filter(
self::$typesNumToConst,
static function (int $num) use ($type): bool {
return ($type & $num) !== 0;
},
ARRAY_FILTER_USE_KEY
);
return implode(' | ', $matchingFlags);
}
/**
* Generates a context's class.
*
* @param array<string, string|array<int, list<string>>> $options the options for this context
* @psalm-param array{
* name: string,
* class: string,
* link: string,
* keywords: array<int, list<string>>
* } $options
*
* @return string
*/
public static function generate($options)
{
$options['keywords'] = static::printWords($options['keywords']);
return sprintf(self::TEMPLATE, $options['name'], $options['class'], $options['link'], $options['keywords']);
}
/**
* Formats context name.
*
* @param string $name name to format
*
* @return string
*/
public static function formatName($name)
{
/* Split name and version */
$parts = [];
if (preg_match('/^(\D+)(\d+)$/', $name, $parts) === 0) {
return $name;
}
/* Format name */
$base = $parts[1];
switch ($base) {
case 'MySql':
$base = 'MySQL';
break;
case 'MariaDb':
$base = 'MariaDB';
break;
}
/* Parse version to array */
$versionString = $parts[2];
if (strlen($versionString) % 2 === 1) {
$versionString = '0' . $versionString;
}
$version = array_map('intval', str_split($versionString, 2));
/* Remove trailing zero */
if ($version[count($version) - 1] === 0) {
$version = array_slice($version, 0, -1);
}
/* Create name */
return $base . ' ' . implode('.', $version);
}
/**
* Builds a test.
*
* Reads the input file, generates the data and writes it back.
*
* @param string $input the input file
* @param string $output the output directory
*
* @return void
*/
public static function build($input, $output)
{
/**
* The directory that contains the input file.
*
* Used to include common files.
*
* @var string
*/
$directory = dirname($input) . '/';
/**
* The name of the file that contains the context.
*/
$file = basename($input);
/**
* The name of the context.
*
* @var string
*/
$name = substr($file, 0, -4);
/**
* The name of the class that defines this context.
*
* @var string
*/
$class = 'Context' . $name;
/**
* The formatted name of this context.
*/
$formattedName = static::formatName($name);
file_put_contents(
$output . '/' . $class . '.php',
static::generate(
[
'name' => $formattedName,
'class' => $class,
'link' => static::$LINKS[$name],
'keywords' => static::readWords(
[
$directory . '_common.txt',
$directory . '_functions' . $file,
$directory . $file,
]
),
]
)
);
}
/**
* Generates recursively all tests preserving the directory structure.
*
* @param string $input the input directory
* @param string $output the output directory
*
* @return void
*/
public static function buildAll($input, $output)
{
$files = scandir($input);
foreach ($files as $file) {
// Skipping current and parent directories.
// Skipping _functions* and _common.txt files
if (($file[0] === '.') || ($file[0] === '_')) {
continue;
}
// Skipping README.md
if ($file === 'README.md') {
continue;
}
// Building the context.
echo sprintf("Building context for %s...\n", $file);
static::build($input . '/' . $file, $output);
}
}
}
@@ -0,0 +1,76 @@
<?php
declare(strict_types=1);
namespace PhpMyAdmin\SqlParser\Tools;
use ReflectionClass;
use ReflectionException;
use Zumba\JsonSerializer\JsonSerializer;
use function in_array;
/**
* Used for .out files generation
*/
class CustomJsonSerializer extends JsonSerializer
{
public const SKIP_PROPERTIES = [
'ALLOWED_KEYWORDS',
'GROUP_OPTIONS',
'END_OPTIONS',
'KEYWORD_PARSERS',
'STATEMENT_PARSERS',
'KEYWORD_NAME_INDICATORS',
'OPERATOR_NAME_INDICATORS',
'DEFAULT_DELIMITER',
'PARSER_METHODS',
'OPTIONS',
'CLAUSES',
'DB_OPTIONS',
'DELIMITERS',
'JOINS',
'FIELDS_OPTIONS',
'LINES_OPTIONS',
'TRIGGER_OPTIONS',
'FUNC_OPTIONS',
'TABLE_OPTIONS',
'FIELD_OPTIONS',
'DATA_TYPE_OPTIONS',
'REFERENCES_OPTIONS',
'KEY_OPTIONS',
'VIEW_OPTIONS',
'EVENT_OPTIONS',
'USER_OPTIONS',
'asciiMap',
];
/**
* Extract the object data
*
* @param object $value
* @param ReflectionClass $ref
* @param string[] $properties
*
* @return array<string,mixed>
*/
protected function extractObjectData($value, $ref, $properties)
{
$data = [];
foreach ($properties as $property) {
if (in_array($property, self::SKIP_PROPERTIES, true)) {
continue;
}
try {
$propRef = $ref->getProperty($property);
$propRef->setAccessible(true);
$data[$property] = $propRef->getValue($value);
} catch (ReflectionException $e) {
$data[$property] = $value->$property;
}
}
return $data;
}
}
@@ -0,0 +1,258 @@
<?php
declare(strict_types=1);
namespace PhpMyAdmin\SqlParser\Tools;
use Exception;
use PhpMyAdmin\SqlParser\Context;
use PhpMyAdmin\SqlParser\Exceptions\LexerException;
use PhpMyAdmin\SqlParser\Exceptions\ParserException;
use PhpMyAdmin\SqlParser\Lexer;
use PhpMyAdmin\SqlParser\Parser;
use PhpMyAdmin\SqlParser\Token;
use function dirname;
use function file_exists;
use function file_get_contents;
use function file_put_contents;
use function in_array;
use function is_dir;
use function json_decode;
use function json_encode;
use function mkdir;
use function print_r;
use function scandir;
use function sprintf;
use function str_contains;
use function str_ends_with;
use function str_replace;
use function strpos;
use function substr;
use const JSON_PRESERVE_ZERO_FRACTION;
use const JSON_PRETTY_PRINT;
use const JSON_UNESCAPED_SLASHES;
use const JSON_UNESCAPED_UNICODE;
/**
* Used for test generation.
*/
class TestGenerator
{
/**
* Generates a test's data.
*
* @param string $query the query to be analyzed
* @param string $type test's type (may be `lexer` or `parser`)
*
* @return array<string, string|Lexer|Parser|array<string, array<int, array<int, int|string|Token|null>>>|null>
*/
public static function generate($query, $type = 'parser')
{
/**
* Lexer used for tokenizing the query.
*/
$lexer = new Lexer($query);
/**
* Parsed used for analyzing the query.
* A new instance of parser is generated only if the test requires.
*
* @var Parser
*/
$parser = $type === 'parser' ? new Parser($lexer->list) : null;
/**
* Lexer's errors.
*
* @var array<int, array<int, int|string>>
*/
$lexerErrors = [];
/**
* Parser's errors.
*/
$parserErrors = [];
// Both the lexer and the parser construct exception for errors.
// Usually, exceptions contain a full stack trace and other details that
// are not required.
// The code below extracts only the relevant information.
// Extracting lexer's errors.
if (! empty($lexer->errors)) {
/** @var LexerException $err */
foreach ($lexer->errors as $err) {
$lexerErrors[] = [
$err->getMessage(),
$err->ch,
$err->pos,
$err->getCode(),
];
}
$lexer->errors = [];
}
// Extracting parser's errors.
if (! empty($parser->errors)) {
/** @var ParserException $err */
foreach ($parser->errors as $err) {
$parserErrors[] = [
$err->getMessage(),
$err->token,
$err->getCode(),
];
}
$parser->errors = [];
}
return [
'query' => $query,
'lexer' => $lexer,
'parser' => $parser,
'errors' => [
'lexer' => $lexerErrors,
'parser' => $parserErrors,
],
];
}
/**
* Builds a test.
*
* Reads the input file, generates the data and writes it back.
*
* @param string $type the type of this test
* @param string $input the input file
* @param string $output the output file
* @param string $debug the debug file
* @param bool $ansi activate quotes ANSI mode
*
* @return void
*/
public static function build($type, $input, $output, $debug = null, $ansi = false)
{
// Support query types: `lexer` / `parser`.
if (! in_array($type, ['lexer', 'parser'])) {
throw new Exception('Unknown test type (expected `lexer` or `parser`).');
}
/**
* The query that is used to generate the test.
*
* @var string
*/
$query = file_get_contents($input);
// There is no point in generating a test without a query.
if (empty($query)) {
throw new Exception('No input query specified.');
}
if ($ansi === true) {
// set ANSI_QUOTES for ansi tests
Context::setMode(Context::SQL_MODE_ANSI_QUOTES);
}
$mariaDbPos = strpos($input, '_mariadb_');
if ($mariaDbPos !== false) {// Keep in sync with TestCase.php
// set context
$mariaDbVersion = (int) substr($input, $mariaDbPos + 9, 6);
Context::load('MariaDb' . $mariaDbVersion);
} else {
// Load the default context to be sure there is no side effects
Context::load('');
}
$test = static::generate($query, $type);
// unset mode, reset to default every time, to be sure
Context::setMode();
$serializer = new CustomJsonSerializer();
// Writing test's data.
$encoded = $serializer->serialize($test);
$encoded = (string) json_encode(
json_decode($encoded),
JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION | JSON_UNESCAPED_SLASHES
);
// Remove the project path from .out file, it changes for each dev
$projectFolder = dirname(__DIR__, 2);// Jump to root
$encoded = str_replace($projectFolder, '<project-root>', $encoded);
file_put_contents($output, $encoded);
// Dumping test's data in human readable format too (if required).
if (empty($debug)) {
return;
}
file_put_contents($debug, print_r($test, true));
}
/**
* Generates recursively all tests preserving the directory structure.
*
* @param string $input the input directory
* @param string $output the output directory
* @param mixed|null $debug
*
* @return void
*/
public static function buildAll($input, $output, $debug = null)
{
$files = scandir($input);
foreach ($files as $file) {
// Skipping current and parent directories.
if (($file === '.') || ($file === '..')) {
continue;
}
// Appending the filename to directories.
$inputFile = $input . '/' . $file;
$outputFile = $output . '/' . $file;
$debugFile = $debug !== null ? $debug . '/' . $file : null;
if (is_dir($inputFile)) {
// Creating required directories to maintain the structure.
// Ignoring errors if the folder structure exists already.
if (! is_dir($outputFile)) {
mkdir($outputFile);
}
if (($debug !== null) && (! is_dir($debugFile))) {
mkdir($debugFile);
}
// Generating tests recursively.
static::buildAll($inputFile, $outputFile, $debugFile);
} elseif (str_ends_with($inputFile, '.in')) {
// Generating file names by replacing `.in` with `.out` and
// `.debug`.
$outputFile = substr($outputFile, 0, -3) . '.out';
if ($debug !== null) {
$debugFile = substr($debugFile, 0, -3) . '.debug';
}
// Building the test.
if (! file_exists($outputFile)) {
echo sprintf("Building test for %s...\n", $inputFile);
static::build(
str_contains($inputFile, 'lex') ? 'lexer' : 'parser',
$inputFile,
$outputFile,
$debugFile,
str_contains($inputFile, 'ansi')
);
} else {
echo sprintf("Test for %s already built!\n", $inputFile);
}
}
}
}
}