resolved conflicts

2026-09-12 11:11:31 +00:00 · 2025-03-31 03:46:05 +03:00
parent b0a69145d7 c7fdd2968a
commit 2dc2287379
6454 changed files with 1520539 additions and 9 deletions
@@ -0,0 +1,147 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+class Tokenizer {
+
+    /**
+     * Token Map for "non-tokens"
+     *
+     * @var array
+     */
+    private $map = [
+        '(' => 'T_OPEN_BRACKET',
+        ')' => 'T_CLOSE_BRACKET',
+        '[' => 'T_OPEN_SQUARE',
+        ']' => 'T_CLOSE_SQUARE',
+        '{' => 'T_OPEN_CURLY',
+        '}' => 'T_CLOSE_CURLY',
+        ';' => 'T_SEMICOLON',
+        '.' => 'T_DOT',
+        ',' => 'T_COMMA',
+        '=' => 'T_EQUAL',
+        '<' => 'T_LT',
+        '>' => 'T_GT',
+        '+' => 'T_PLUS',
+        '-' => 'T_MINUS',
+        '*' => 'T_MULT',
+        '/' => 'T_DIV',
+        '?' => 'T_QUESTION_MARK',
+        '!' => 'T_EXCLAMATION_MARK',
+        ':' => 'T_COLON',
+        '"' => 'T_DOUBLE_QUOTES',
+        '@' => 'T_AT',
+        '&' => 'T_AMPERSAND',
+        '%' => 'T_PERCENT',
+        '|' => 'T_PIPE',
+        '$' => 'T_DOLLAR',
+        '^' => 'T_CARET',
+        '~' => 'T_TILDE',
+        '`' => 'T_BACKTICK'
+    ];
+
+    public function parse(string $source): TokenCollection {
+        $result = new TokenCollection();
+
+        if ($source === '') {
+            return $result;
+        }
+
+        $tokens = \token_get_all($source);
+
+        $lastToken = new Token(
+            $tokens[0][2],
+            'Placeholder',
+            ''
+        );
+
+        foreach ($tokens as $pos => $tok) {
+            if (\is_string($tok)) {
+                $token = new Token(
+                    $lastToken->getLine(),
+                    $this->map[$tok],
+                    $tok
+                );
+                $result->addToken($token);
+                $lastToken = $token;
+
+                continue;
+            }
+
+            $line   = $tok[2];
+            $values = \preg_split('/\R+/Uu', $tok[1]);
+
+            if (!$values) {
+                $result->addToken(
+                    new Token(
+                        $line,
+                        \token_name($tok[0]),
+                        '{binary data}'
+                    )
+                );
+
+                continue;
+            }
+
+            foreach ($values as $v) {
+                $token = new Token(
+                    $line,
+                    \token_name($tok[0]),
+                    $v
+                );
+                $lastToken = $token;
+                $line++;
+
+                if ($v === '') {
+                    continue;
+                }
+
+                $result->addToken($token);
+            }
+        }
+
+        return $this->fillBlanks($result, $lastToken->getLine());
+    }
+
+    private function fillBlanks(TokenCollection $tokens, int $maxLine): TokenCollection {
+        $prev = new Token(
+            0,
+            'Placeholder',
+            ''
+        );
+
+        $final = new TokenCollection();
+
+        foreach ($tokens as $token) {
+            $gap = $token->getLine() - $prev->getLine();
+
+            while ($gap > 1) {
+                $linebreak = new Token(
+                    $prev->getLine() + 1,
+                    'T_WHITESPACE',
+                    ''
+                );
+                $final->addToken($linebreak);
+                $prev = $linebreak;
+                $gap--;
+            }
+
+            $final->addToken($token);
+            $prev = $token;
+        }
+
+        $gap = $maxLine - $prev->getLine();
+
+        while ($gap > 0) {
+            $linebreak = new Token(
+                $prev->getLine() + 1,
+                'T_WHITESPACE',
+                ''
+            );
+            $final->addToken($linebreak);
+            $prev = $linebreak;
+            $gap--;
+        }
+
+        return $final;
+    }
+}
@@ -0,0 +1,79 @@
+<?php declare(strict_types = 1);
+namespace TheSeer\Tokenizer;
+
+use DOMDocument;
+
+class XMLSerializer {
+
+    /** @var \XMLWriter */
+    private $writer;
+
+    /** @var Token */
+    private $previousToken;
+
+    /** @var NamespaceUri */
+    private $xmlns;
+
+    /**
+     * XMLSerializer constructor.
+     *
+     * @param NamespaceUri $xmlns
+     */
+    public function __construct(?NamespaceUri $xmlns = null) {
+        if ($xmlns === null) {
+            $xmlns = new NamespaceUri('https://github.com/theseer/tokenizer');
+        }
+        $this->xmlns = $xmlns;
+    }
+
+    public function toDom(TokenCollection $tokens): DOMDocument {
+        $dom                     = new DOMDocument();
+        $dom->preserveWhiteSpace = false;
+        $dom->loadXML($this->toXML($tokens));
+
+        return $dom;
+    }
+
+    public function toXML(TokenCollection $tokens): string {
+        $this->writer = new \XMLWriter();
+        $this->writer->openMemory();
+        $this->writer->setIndent(true);
+        $this->writer->startDocument();
+        $this->writer->startElement('source');
+        $this->writer->writeAttribute('xmlns', $this->xmlns->asString());
+
+        if (\count($tokens) > 0) {
+            $this->writer->startElement('line');
+            $this->writer->writeAttribute('no', '1');
+
+            $this->previousToken = $tokens[0];
+
+            foreach ($tokens as $token) {
+                $this->addToken($token);
+            }
+        }
+
+        $this->writer->endElement();
+        $this->writer->endElement();
+        $this->writer->endDocument();
+
+        return $this->writer->outputMemory();
+    }
+
+    private function addToken(Token $token): void {
+        if ($this->previousToken->getLine() < $token->getLine()) {
+            $this->writer->endElement();
+
+            $this->writer->startElement('line');
+            $this->writer->writeAttribute('no', (string)$token->getLine());
+            $this->previousToken = $token;
+        }
+
+        if ($token->getValue() !== '') {
+            $this->writer->startElement('token');
+            $this->writer->writeAttribute('name', $token->getName());
+            $this->writer->writeRaw(\htmlspecialchars($token->getValue(), \ENT_NOQUOTES | \ENT_DISALLOWED | \ENT_XML1));
+            $this->writer->endElement();
+        }
+    }
+}