mirror of
https://gitlab.com/signalytic/client-external/streamline/streamline-emr.git
synced 2026-09-12 03:01:32 +00:00
resolved conflicts
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to Tokenizer are documented in this file using the [Keep a CHANGELOG](http://keepachangelog.com/) principles.
|
||||
|
||||
## [1.2.3] - 2024-03-03
|
||||
|
||||
### Changed
|
||||
|
||||
* Do not use implicitly nullable parameters
|
||||
|
||||
## [1.2.2] - 2023-11-20
|
||||
|
||||
### Fixed
|
||||
|
||||
* [#18](https://github.com/theseer/tokenizer/issues/18): Tokenizer fails on protobuf metadata files
|
||||
|
||||
|
||||
## [1.2.1] - 2021-07-28
|
||||
|
||||
### Fixed
|
||||
|
||||
* [#13](https://github.com/theseer/tokenizer/issues/13): Fatal error when tokenizing files that contain only a single empty line
|
||||
|
||||
|
||||
## [1.2.0] - 2020-07-13
|
||||
|
||||
This release is now PHP 8.0 compliant.
|
||||
|
||||
### Fixed
|
||||
|
||||
* Whitespace handling in general (only noticable in the intermediate `TokenCollection`) is now consitent
|
||||
|
||||
### Changed
|
||||
|
||||
* Updated `Tokenizer` to deal with changed whitespace handling in PHP 8.0
|
||||
The XMLSerializer was unaffected.
|
||||
|
||||
|
||||
## [1.1.3] - 2019-06-14
|
||||
|
||||
### Changed
|
||||
|
||||
* Ensure XMLSerializer can deal with empty token collections
|
||||
|
||||
### Fixed
|
||||
|
||||
* [#2](https://github.com/theseer/tokenizer/issues/2): Fatal error in infection / phpunit
|
||||
|
||||
|
||||
## [1.1.2] - 2019-04-04
|
||||
|
||||
### Changed
|
||||
|
||||
* Reverted PHPUnit 8 test update to stay PHP 7.0 compliant
|
||||
|
||||
|
||||
## [1.1.1] - 2019-04-03
|
||||
|
||||
### Fixed
|
||||
|
||||
* [#1](https://github.com/theseer/tokenizer/issues/1): Empty file causes invalid array read
|
||||
|
||||
### Changed
|
||||
|
||||
* Tests should now be PHPUnit 8 compliant
|
||||
|
||||
|
||||
## [1.1.0] - 2017-04-07
|
||||
|
||||
### Added
|
||||
|
||||
* Allow use of custom namespace for XML serialization
|
||||
|
||||
|
||||
## [1.0.0] - 2017-04-05
|
||||
|
||||
Initial Release
|
||||
|
||||
[1.2.3]: https://github.com/theseer/tokenizer/compare/1.2.2...1.2.3
|
||||
[1.2.2]: https://github.com/theseer/tokenizer/compare/1.2.1...1.2.2
|
||||
[1.2.1]: https://github.com/theseer/tokenizer/compare/1.2.0...1.2.1
|
||||
[1.2.0]: https://github.com/theseer/tokenizer/compare/1.1.3...1.2.0
|
||||
[1.1.3]: https://github.com/theseer/tokenizer/compare/1.1.2...1.1.3
|
||||
[1.1.2]: https://github.com/theseer/tokenizer/compare/1.1.1...1.1.2
|
||||
[1.1.1]: https://github.com/theseer/tokenizer/compare/1.1.0...1.1.1
|
||||
[1.1.0]: https://github.com/theseer/tokenizer/compare/1.0.0...1.1.0
|
||||
[1.0.0]: https://github.com/theseer/tokenizer/compare/b2493e57de80c1b7414219b28503fa5c6b4d0a98...1.0.0
|
||||
@@ -0,0 +1,147 @@
|
||||
<?php declare(strict_types = 1);
|
||||
namespace TheSeer\Tokenizer;
|
||||
|
||||
class Tokenizer {
|
||||
|
||||
/**
|
||||
* Token Map for "non-tokens"
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
private $map = [
|
||||
'(' => 'T_OPEN_BRACKET',
|
||||
')' => 'T_CLOSE_BRACKET',
|
||||
'[' => 'T_OPEN_SQUARE',
|
||||
']' => 'T_CLOSE_SQUARE',
|
||||
'{' => 'T_OPEN_CURLY',
|
||||
'}' => 'T_CLOSE_CURLY',
|
||||
';' => 'T_SEMICOLON',
|
||||
'.' => 'T_DOT',
|
||||
',' => 'T_COMMA',
|
||||
'=' => 'T_EQUAL',
|
||||
'<' => 'T_LT',
|
||||
'>' => 'T_GT',
|
||||
'+' => 'T_PLUS',
|
||||
'-' => 'T_MINUS',
|
||||
'*' => 'T_MULT',
|
||||
'/' => 'T_DIV',
|
||||
'?' => 'T_QUESTION_MARK',
|
||||
'!' => 'T_EXCLAMATION_MARK',
|
||||
':' => 'T_COLON',
|
||||
'"' => 'T_DOUBLE_QUOTES',
|
||||
'@' => 'T_AT',
|
||||
'&' => 'T_AMPERSAND',
|
||||
'%' => 'T_PERCENT',
|
||||
'|' => 'T_PIPE',
|
||||
'$' => 'T_DOLLAR',
|
||||
'^' => 'T_CARET',
|
||||
'~' => 'T_TILDE',
|
||||
'`' => 'T_BACKTICK'
|
||||
];
|
||||
|
||||
public function parse(string $source): TokenCollection {
|
||||
$result = new TokenCollection();
|
||||
|
||||
if ($source === '') {
|
||||
return $result;
|
||||
}
|
||||
|
||||
$tokens = \token_get_all($source);
|
||||
|
||||
$lastToken = new Token(
|
||||
$tokens[0][2],
|
||||
'Placeholder',
|
||||
''
|
||||
);
|
||||
|
||||
foreach ($tokens as $pos => $tok) {
|
||||
if (\is_string($tok)) {
|
||||
$token = new Token(
|
||||
$lastToken->getLine(),
|
||||
$this->map[$tok],
|
||||
$tok
|
||||
);
|
||||
$result->addToken($token);
|
||||
$lastToken = $token;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$line = $tok[2];
|
||||
$values = \preg_split('/\R+/Uu', $tok[1]);
|
||||
|
||||
if (!$values) {
|
||||
$result->addToken(
|
||||
new Token(
|
||||
$line,
|
||||
\token_name($tok[0]),
|
||||
'{binary data}'
|
||||
)
|
||||
);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($values as $v) {
|
||||
$token = new Token(
|
||||
$line,
|
||||
\token_name($tok[0]),
|
||||
$v
|
||||
);
|
||||
$lastToken = $token;
|
||||
$line++;
|
||||
|
||||
if ($v === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$result->addToken($token);
|
||||
}
|
||||
}
|
||||
|
||||
return $this->fillBlanks($result, $lastToken->getLine());
|
||||
}
|
||||
|
||||
private function fillBlanks(TokenCollection $tokens, int $maxLine): TokenCollection {
|
||||
$prev = new Token(
|
||||
0,
|
||||
'Placeholder',
|
||||
''
|
||||
);
|
||||
|
||||
$final = new TokenCollection();
|
||||
|
||||
foreach ($tokens as $token) {
|
||||
$gap = $token->getLine() - $prev->getLine();
|
||||
|
||||
while ($gap > 1) {
|
||||
$linebreak = new Token(
|
||||
$prev->getLine() + 1,
|
||||
'T_WHITESPACE',
|
||||
''
|
||||
);
|
||||
$final->addToken($linebreak);
|
||||
$prev = $linebreak;
|
||||
$gap--;
|
||||
}
|
||||
|
||||
$final->addToken($token);
|
||||
$prev = $token;
|
||||
}
|
||||
|
||||
$gap = $maxLine - $prev->getLine();
|
||||
|
||||
while ($gap > 0) {
|
||||
$linebreak = new Token(
|
||||
$prev->getLine() + 1,
|
||||
'T_WHITESPACE',
|
||||
''
|
||||
);
|
||||
$final->addToken($linebreak);
|
||||
$prev = $linebreak;
|
||||
$gap--;
|
||||
}
|
||||
|
||||
return $final;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
<?php declare(strict_types = 1);
|
||||
namespace TheSeer\Tokenizer;
|
||||
|
||||
use DOMDocument;
|
||||
|
||||
class XMLSerializer {
|
||||
|
||||
/** @var \XMLWriter */
|
||||
private $writer;
|
||||
|
||||
/** @var Token */
|
||||
private $previousToken;
|
||||
|
||||
/** @var NamespaceUri */
|
||||
private $xmlns;
|
||||
|
||||
/**
|
||||
* XMLSerializer constructor.
|
||||
*
|
||||
* @param NamespaceUri $xmlns
|
||||
*/
|
||||
public function __construct(?NamespaceUri $xmlns = null) {
|
||||
if ($xmlns === null) {
|
||||
$xmlns = new NamespaceUri('https://github.com/theseer/tokenizer');
|
||||
}
|
||||
$this->xmlns = $xmlns;
|
||||
}
|
||||
|
||||
public function toDom(TokenCollection $tokens): DOMDocument {
|
||||
$dom = new DOMDocument();
|
||||
$dom->preserveWhiteSpace = false;
|
||||
$dom->loadXML($this->toXML($tokens));
|
||||
|
||||
return $dom;
|
||||
}
|
||||
|
||||
public function toXML(TokenCollection $tokens): string {
|
||||
$this->writer = new \XMLWriter();
|
||||
$this->writer->openMemory();
|
||||
$this->writer->setIndent(true);
|
||||
$this->writer->startDocument();
|
||||
$this->writer->startElement('source');
|
||||
$this->writer->writeAttribute('xmlns', $this->xmlns->asString());
|
||||
|
||||
if (\count($tokens) > 0) {
|
||||
$this->writer->startElement('line');
|
||||
$this->writer->writeAttribute('no', '1');
|
||||
|
||||
$this->previousToken = $tokens[0];
|
||||
|
||||
foreach ($tokens as $token) {
|
||||
$this->addToken($token);
|
||||
}
|
||||
}
|
||||
|
||||
$this->writer->endElement();
|
||||
$this->writer->endElement();
|
||||
$this->writer->endDocument();
|
||||
|
||||
return $this->writer->outputMemory();
|
||||
}
|
||||
|
||||
private function addToken(Token $token): void {
|
||||
if ($this->previousToken->getLine() < $token->getLine()) {
|
||||
$this->writer->endElement();
|
||||
|
||||
$this->writer->startElement('line');
|
||||
$this->writer->writeAttribute('no', (string)$token->getLine());
|
||||
$this->previousToken = $token;
|
||||
}
|
||||
|
||||
if ($token->getValue() !== '') {
|
||||
$this->writer->startElement('token');
|
||||
$this->writer->writeAttribute('name', $token->getName());
|
||||
$this->writer->writeRaw(\htmlspecialchars($token->getValue(), \ENT_NOQUOTES | \ENT_DISALLOWED | \ENT_XML1));
|
||||
$this->writer->endElement();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user