php Symfony-CS-Tokenizer-Tokens类(方法)实例源码

/**
  * Replace occurrences of the name of the classy element by "self" (if possible).
  *
  * @param Tokens $tokens
  * @param string $name
  * @param int    $startIndex
  * @param int    $endIndex
  */
 private function replaceNameOccurrences(Tokens $tokens, $name, $startIndex, $endIndex)
 {
     $tokensAnalyzer = new TokensAnalyzer($tokens);
     for ($i = $startIndex; $i < $endIndex; ++$i) {
         $token = $tokens[$i];
         // skip lambda functions (PHP < 5.4 compatibility)
         if ($token->isGivenKind(T_FUNCTION) && $tokensAnalyzer->isLambda($i)) {
             $i = $tokens->getNextTokenOfKind($i, array('{'));
             $i = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_CURLY_BRACE, $i);
             continue;
         }
         if (!$token->equals(array(T_STRING, $name), false)) {
             continue;
         }
         $prevToken = $tokens[$tokens->getPrevMeaningfulToken($i)];
         $nextToken = $tokens[$tokens->getNextMeaningfulToken($i)];
         // skip tokens that are part of a fully qualified name
         if ($prevToken->isGivenKind(T_NS_SEPARATOR) || $nextToken->isGivenKind(T_NS_SEPARATOR)) {
             continue;
         }
         if ($prevToken->isGivenKind(array(T_INSTANCEOF, T_NEW)) || $nextToken->isGivenKind(T_PAAMAYIM_NEKUDOTAYIM)) {
             $token->setContent('self');
         }
     }
 }

private function getNewOrder(array $uses, Tokens $tokens)
 {
     $uses = array_reverse($uses);
     $indexes = array();
     $originalIndexes = array();
     foreach ($uses as $index) {
         $endIndex = $tokens->getNextTokenOfKind($index, array(';'));
         $startIndex = $tokens->getTokenNotOfKindSibling($index + 1, 1, array(array(T_WHITESPACE)));
         $namespace = '';
         $index = $startIndex;
         while ($index <= $endIndex) {
             $token = $tokens[$index];
             /** @var Token $token */
             if ($token->equals(',') || $index === $endIndex) {
                 $indexes[$startIndex] = array($namespace, $startIndex, $index - 1);
                 $originalIndexes[] = $startIndex;
                 $namespace = '';
                 $nextPartIndex = $tokens->getTokenNotOfKindSibling($index, 1, array(array(','), array(T_WHITESPACE)));
                 $startIndex = $nextPartIndex;
                 $index = $nextPartIndex;
                 continue;
             }
             $namespace .= $token->getContent();
             ++$index;
         }
     }
     uasort($indexes, 'self::sortingCallBack');
     $i = -1;
     $usesOrder = array();
     // Loop trough the index but use original index order
     foreach ($indexes as $v) {
         $usesOrder[$originalIndexes[++$i]] = $v;
     }
     return $usesOrder;
 }

/**
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     /** @var $token \Symfony\CS\Tokenizer\Token */
     foreach ($tokens->findGivenKind(T_STRING) as $index => $token) {
         // skip expressions without parameters list
         $nextToken = $tokens[$tokens->getNextMeaningfulToken($index)];
         if (!$nextToken->equals('(')) {
             continue;
         }
         // skip expressions which are not function reference
         $prevTokenIndex = $tokens->getPrevMeaningfulToken($index);
         $prevToken = $tokens[$prevTokenIndex];
         if ($prevToken->isGivenKind(array(T_DOUBLE_COLON, T_NEW, T_OBJECT_OPERATOR, T_FUNCTION))) {
             continue;
         }
         // handle function reference with namespaces
         if ($prevToken->isGivenKind(array(T_NS_SEPARATOR))) {
             $twicePrevTokenIndex = $tokens->getPrevMeaningfulToken($prevTokenIndex);
             $twicePrevToken = $tokens[$twicePrevTokenIndex];
             if ($twicePrevToken->isGivenKind(array(T_DOUBLE_COLON, T_NEW, T_OBJECT_OPERATOR, T_FUNCTION, T_STRING, CT_NAMESPACE_OPERATOR))) {
                 continue;
             }
         }
         // check mapping hit
         $tokenContent = strtolower($token->getContent());
         if (!isset(self::$aliases[$tokenContent])) {
             continue;
         }
         $token->setContent(self::$aliases[$tokenContent]);
     }
 }

/**
  * Inject into the text placeholders of candidates of vertical alignment.
  *
  * @param Tokens $tokens
  * @param int    $startAt
  * @param int    $endAt
  *
  * @return array($code, $context_counter)
  */
 private function injectAlignmentPlaceholders(Tokens $tokens, $startAt, $endAt)
 {
     for ($index = $startAt; $index < $endAt; ++$index) {
         $token = $tokens[$index];
         if ($token->isGivenKind(array(T_FOREACH, T_FOR, T_WHILE, T_IF, T_SWITCH))) {
             $index = $tokens->getNextMeaningfulToken($index);
             $index = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $index);
             continue;
         }
         if ($token->isGivenKind(T_ARRAY)) {
             // don't use "$tokens->isArray()" here, short arrays are handled in the next case
             $from = $tokens->getNextMeaningfulToken($index);
             $until = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $from);
             $index = $until;
             ++$this->deepestLevel;
             ++$this->currentLevel;
             $this->injectAlignmentPlaceholders($tokens, $from, $until);
             --$this->currentLevel;
             continue;
         }
         if ($token->isGivenKind(CT_ARRAY_SQUARE_BRACE_OPEN)) {
             $prevToken = $tokens[$tokens->getPrevMeaningfulToken($index)];
             if ($prevToken->isGivenKind(array(T_STRING, T_VARIABLE))) {
                 continue;
             }
             $from = $index;
             $until = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, $from);
             $index = $until;
             ++$this->deepestLevel;
             ++$this->currentLevel;
             $this->injectAlignmentPlaceholders($tokens, $from + 1, $until - 1);
             --$this->currentLevel;
             continue;
         }
         if ($token->isGivenKind(T_DOUBLE_ARROW)) {
             $tokenContent = sprintf(self::ALIGNABLE_PLACEHOLDER, $this->currentLevel) . $token->getContent();
             $nextToken = $tokens[$index + 1];
             if (!$nextToken->isWhitespace()) {
                 $tokenContent .= ' ';
             } elseif ($nextToken->isWhitespace(" \t")) {
                 $nextToken->setContent(' ');
             }
             $token->setContent($tokenContent);
             continue;
         }
         if ($token->equals(';')) {
             ++$this->deepestLevel;
             ++$this->currentLevel;
             continue;
         }
         if ($token->equals(',')) {
             for ($i = $index; $i < $endAt - 1; ++$i) {
                 if ($tokens[$i + 1]->isGivenKind(array(T_ARRAY, CT_ARRAY_SQUARE_BRACE_OPEN)) || false !== strpos($tokens[$i - 1]->getContent(), "\n")) {
                     break;
                 }
                 ++$index;
             }
         }
     }
 }

/**
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     // ignore non-monolithic files
     if (!$tokens->isMonolithicPhp()) {
         return;
     }
     // ignore files with short open tag
     if (!$tokens[0]->isGivenKind(T_OPEN_TAG)) {
         return;
     }
     $newlineFound = false;
     foreach ($tokens as $token) {
         if ($token->isWhitespace("\n")) {
             $newlineFound = true;
             break;
         }
     }
     // ignore one-line files
     if (!$newlineFound) {
         return;
     }
     $token = $tokens[0];
     if (false === strpos($token->getContent(), "\n")) {
         $token->setContent(rtrim($token->getContent()) . "\n");
     }
     if (!$tokens[1]->isWhitespace("\n")) {
         $tokens->insertAt(1, new Token(array(T_WHITESPACE, "\n")));
     }
 }

private function getNamespaceUseDeclarations(Tokens $tokens, array $useIndexes)
 {
     $uses = array();
     foreach ($useIndexes as $index) {
         $declarationEndIndex = $tokens->getNextTokenOfKind($index, array(';'));
         $declarationContent = $tokens->generatePartialCode($index + 1, $declarationEndIndex - 1);
         // ignore multiple use statements like: `use BarB, BarC as C, BarD;`
         // that should be split into few separate statements
         if (false !== strpos($declarationContent, ',')) {
             continue;
         }
         $declarationParts = preg_split('/\\s+as\\s+/i', $declarationContent);
         if (1 === count($declarationParts)) {
             $fullName = $declarationContent;
             $declarationParts = explode('\\', $fullName);
             $shortName = end($declarationParts);
             $aliased = false;
         } else {
             $fullName = $declarationParts[0];
             $shortName = $declarationParts[1];
             $declarationParts = explode('\\', $fullName);
             $aliased = $shortName !== end($declarationParts);
         }
         $shortName = trim($shortName);
         $uses[$shortName] = array('shortName' => $shortName, 'fullName' => trim($fullName), 'aliased' => $aliased, 'declarationStart' => $index, 'declarationEnd' => $declarationEndIndex);
     }
     return $uses;
 }

/**
  * Replace all `else if` (T_ELSE T_IF) with `elseif` (T_ELSEIF).
  *
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     foreach ($tokens as $index => $token) {
         if (!$token->isGivenKind(T_ELSE)) {
             continue;
         }
         $nextIndex = $tokens->getNextNonWhitespace($index);
         $nextToken = $tokens[$nextIndex];
         // if next meaning token is not T_IF - continue searching, this is not the case for fixing
         if (!$nextToken->isGivenKind(T_IF)) {
             continue;
         }
         // now we have T_ELSE following by T_IF so we could fix this
         // 1. clear whitespaces between T_ELSE and T_IF
         $tokens[$index + 1]->clear();
         // 2. change token from T_ELSE into T_ELSEIF
         $tokens->overrideAt($index, array(T_ELSEIF, 'elseif'));
         // 3. clear succeeding T_IF
         $nextToken->clear();
     }
     # handle `T_ELSE T_WHITESPACE T_IF` treated as single `T_ELSEIF` by HHVM
     # see https://github.com/facebook/hhvm/issues/4796
     if (defined('HHVM_VERSION')) {
         foreach ($tokens->findGivenKind(T_ELSEIF) as $token) {
             $token->setContent('elseif');
         }
     }
 }

/**
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     // Checks if specific statements are set and uses them in this case.
     $loops = array_intersect_key(self::$loops, array_flip($this->controlStatements));
     foreach ($tokens as $index => $token) {
         if (!$token->equals('(')) {
             continue;
         }
         $blockStartIndex = $index;
         $index = $tokens->getPrevMeaningfulToken($index);
         $token = $tokens[$index];
         foreach ($loops as $loop) {
             if (!$token->isGivenKind($loop['lookupTokens'])) {
                 continue;
             }
             $blockEndIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $blockStartIndex);
             $blockEndNextIndex = $tokens->getNextMeaningfulToken($blockEndIndex);
             if (!$tokens[$blockEndNextIndex]->equalsAny($loop['neededSuccessors'])) {
                 continue;
             }
             if ($tokens[$blockStartIndex - 1]->isWhitespace() || $tokens[$blockStartIndex - 1]->isComment()) {
                 $this->clearParenthesis($tokens, $blockStartIndex);
             } else {
                 // Adds a space to prevent broken code like `return2`.
                 $tokens->overrideAt($blockStartIndex, array(T_WHITESPACE, ' '));
             }
             $this->clearParenthesis($tokens, $blockEndIndex);
         }
     }
 }

/**
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     foreach ($tokens as $index => $token) {
         if (!$token->isGivenKind(array(T_CASE, T_DEFAULT))) {
             continue;
         }
         $ternariesCount = 0;
         for ($colonIndex = $index + 1;; ++$colonIndex) {
             // We have to skip ternary case for colons.
             if ($tokens[$colonIndex]->equals('?')) {
                 ++$ternariesCount;
             }
             if ($tokens[$colonIndex]->equals(':')) {
                 if (0 === $ternariesCount) {
                     break;
                 }
                 --$ternariesCount;
             }
         }
         $valueIndex = $tokens->getPrevNonWhitespace($colonIndex);
         if (2 + $valueIndex === $colonIndex) {
             $tokens[$valueIndex + 1]->clear();
         }
     }
 }

/**
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     static $nativeFunctionNames = null;
     if (null === $nativeFunctionNames) {
         $nativeFunctionNames = $this->getNativeFunctionNames();
     }
     for ($index = 0, $count = $tokens->count(); $index < $count; ++$index) {
         // test if we are at a function all
         if (!$tokens[$index]->isGivenKind(T_STRING)) {
             continue;
         }
         $next = $tokens->getNextMeaningfulToken($index);
         if (!$tokens[$next]->equals('(')) {
             $index = $next;
             continue;
         }
         $functionNamePrefix = $tokens->getPrevMeaningfulToken($index);
         if ($tokens[$functionNamePrefix]->isGivenKind(array(T_DOUBLE_COLON, T_NEW, T_OBJECT_OPERATOR, T_FUNCTION))) {
             continue;
         }
         // do not though the function call if it is to a function in a namespace other than the default
         if ($tokens[$functionNamePrefix]->isGivenKind(T_NS_SEPARATOR) && $tokens[$tokens->getPrevMeaningfulToken($functionNamePrefix)]->isGivenKind(T_STRING)) {
             continue;
         }
         // test if the function call is to a native PHP function
         $lower = strtolower($tokens[$index]->getContent());
         if (!array_key_exists($lower, $nativeFunctionNames)) {
             continue;
         }
         $tokens[$index]->setContent($nativeFunctionNames[$lower]);
         $index = $next;
     }
 }

private function fixFunction(Tokens $tokens, $functionIndex, array $functionParams)
{
$startBraceIndex = $tokens->getNextTokenOfKind($functionIndex, array('('));
$endBraceIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $startBraceIndex);
$commaCounter = 0;
$sawParameter = false;

for ($index = $startBraceIndex + 1; $index < $endBraceIndex; ++$index) {
$token = $tokens[$index];

if (!$token->isWhitespace() && !$token->isComment()) {
$sawParameter = true;
}

if ($token->equals('(')) {
$index = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $index);
continue;
}

if ($token->equals('[')) {
$index = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_SQUARE_BRACE, $index);
continue;
}

if ($token->equals(',')) {
++$commaCounter;
continue;
}
}

$functionParamsQuantity = count($functionParams);
$paramsQuantity = ($sawParameter ? 1 : 0) + $commaCounter;

if ($paramsQuantity === $functionParamsQuantity) {
return;
}

$tokensToInsert = array();
for ($i = $paramsQuantity; $i < $functionParamsQuantity; ++$i) {

 if (!$functionParams[$i]) {
return;
}

$tokensToInsert[] = new Token(',');
$tokensToInsert[] = new Token(array(T_WHITESPACE, ' '));

if (!is_array($functionParams[$i])) {
$tokensToInsert[] = clone $functionParams[$i];
continue;
}

foreach ($functionParams[$i] as $param) {
$tokensToInsert[] = clone $param;
}
}

$beforeEndBraceIndex = $tokens->getPrevNonWhitespace($endBraceIndex, array());
$tokens->insertAt($beforeEndBraceIndex + 1, $tokensToInsert);
}

/**
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     $functionyTokens = $this->getFunctionyTokenKinds();
     $languageConstructionTokens = $this->getLanguageConstructionTokenKinds();
     foreach ($tokens as $index => $token) {
         // looking for start brace
         if (!$token->equals('(')) {
             continue;
         }
         // last non-whitespace token
         $lastTokenIndex = $tokens->getPrevNonWhitespace($index);
         if (null === $lastTokenIndex) {
             continue;
         }
         // check for ternary operator
         $endParenthesisIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $index);
         $nextNonWhiteSpace = $tokens->getNextMeaningfulToken($endParenthesisIndex);
         if (!empty($nextNonWhiteSpace) && $tokens[$nextNonWhiteSpace]->equals('?') && $tokens[$lastTokenIndex]->isGivenKind($languageConstructionTokens)) {
             continue;
         }
         // check if it is a function call
         if ($tokens[$lastTokenIndex]->isGivenKind($functionyTokens)) {
             $this->fixFunctionCall($tokens, $index);
         }
     }
 }

/**
  * @param Tokens|Token[] $tokens
  *
  * @return void
  */
 protected function fixContent(Tokens $tokens)
 {
     $wrongTokens = [T_FUNCTION, T_OBJECT_OPERATOR, T_NEW];
     foreach ($tokens as $index => $token) {
         $tokenContent = strtolower($token->getContent());
         if (empty($tokenContent) || !isset(self::$matching[$tokenContent])) {
             continue;
         }
         $prevIndex = $tokens->getPrevNonWhitespace($index);
         if (in_array($tokens[$prevIndex]->getId(), $wrongTokens, true)) {
             continue;
         }
         $openingBrace = $tokens->getNextMeaningfulToken($index);
         if ($tokens[$openingBrace]->getContent() !== '(') {
             continue;
         }
         $closingBrace = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $openingBrace);
         // Skip for non-trivial cases
         for ($i = $openingBrace + 1; $i < $closingBrace; ++$i) {
             if ($tokens[$i]->equals(',')) {
                 continue 2;
             }
         }
         $cast = '(' . self::$matching[$tokenContent] . ')';
         $tokens[$index]->setContent($cast);
         $tokens[$openingBrace]->setContent('');
         $tokens[$closingBrace]->setContent('');
     }
 }

/**
  * Inject into the text placeholders of candidates of vertical alignment.
  *
  * @param Tokens $tokens
  */
 private function injectAlignmentPlaceholders(Tokens $tokens)
 {
     $deepestLevel = 0;
     $limit = $tokens->count();
     for ($index = 0; $index < $limit; ++$index) {
         $token = $tokens[$index];
         if ($token->equals('=')) {
             $token->setContent(sprintf(self::ALIGNABLE_PLACEHOLDER, $deepestLevel) . $token->getContent());
             continue;
         }
         if ($token->isGivenKind(T_FUNCTION)) {
             ++$deepestLevel;
             continue;
         }
         if ($token->equals('(')) {
             $index = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_PARENTHESIS_BRACE, $index);
             continue;
         }
         if ($token->equals('[')) {
             $index = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_INDEX_SQUARE_BRACE, $index);
             continue;
         }
         if ($token->isGivenKind(CT_ARRAY_SQUARE_BRACE_OPEN)) {
             $index = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ARRAY_SQUARE_BRACE, $index);
             continue;
         }
     }
     $this->deepestLevel = $deepestLevel;
 }

public function process(Tokens $tokens)
{
foreach ($tokens as $index => $token) {
if (!$token->equals('$')) {
continue;
}

$openIndex = $tokens->getNextMeaningfulToken($index);

if (null === $openIndex) {
continue;
}

$openToken = $tokens[$openIndex];

if (!$openToken->equals('{')) {
continue;
}

$closeIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_CURLY_BRACE, $openIndex);
$closeToken = $tokens[$closeIndex];

$openToken->override(array(CT_DYNAMIC_VAR_BRACE_OPEN, '{', $openToken->getLine()));
$closeToken->override(array(CT_DYNAMIC_VAR_BRACE_CLOSE, '}', $closeToken->getLine()));
}
}

/**
  * {@inheritdoc}
  */
 public function process(Tokens $tokens)
 {
     foreach ($tokens->findGivenKind(T_DOLLAR_OPEN_CURLY_BRACES) as $index => $token) {
         $nextIndex = $tokens->getNextTokenOfKind($index, array('}'));
         $tokens[$nextIndex]->override(array(CT_DOLLAR_CLOSE_CURLY_BRACES, '}', $tokens[$nextIndex]->getLine()));
     }
 }

/**
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     $namespace = false;
     $classyName = null;
     $classyIndex = 0;
     foreach ($tokens as $index => $token) {
         if ($token->isGivenKind(T_NAMESPACE)) {
             if (false !== $namespace) {
                 return;
             }
             $namespace = true;
         } elseif ($token->isClassy()) {
             if (null !== $classyName) {
                 return;
             }
             $classyIndex = $tokens->getNextMeaningfulToken($index);
             $classyName = $tokens[$classyIndex]->getContent();
         }
     }
     if (null === $classyName) {
         return;
     }
     if (false !== $namespace) {
         $filename = basename(strtr($file->getRealPath(), '\\', '/'), '.php');
         if ($classyName !== $filename) {
             $tokens[$classyIndex]->setContent($filename);
         }
     } else {
         $normClass = strtr($classyName, '_', '/');
         $filename = substr(strtr($file->getRealPath(), '\\', '/'), -strlen($normClass) - 4, -4);
         if ($normClass !== $filename && strtolower($normClass) === strtolower($filename)) {
             $tokens[$classyIndex]->setContent(strtr($filename, '/', '_'));
         }
     }
 }

/**
  * @param Tokens|Token[] $tokens
  *
  * @return void
  */
 protected function fixContent(Tokens $tokens)
 {
     $wrongTokens = [T_FUNCTION, T_OBJECT_OPERATOR, T_NEW];
     foreach ($tokens as $index => $token) {
         $tokenContent = strtolower($token->getContent());
         if (strtolower($tokenContent) !== 'php_sapi_name') {
             continue;
         }
         $openingBrace = $tokens->getNextMeaningfulToken($index);
         if ($openingBrace === null || $tokens[$openingBrace]->getContent() !== '(') {
             continue;
         }
         $closingBrace = $tokens->getNextMeaningfulToken($openingBrace);
         if ($closingBrace === null || $tokens[$closingBrace]->getContent() !== ')') {
             continue;
         }
         $prevIndex = $tokens->getPrevNonWhitespace($index);
         if ($prevIndex === null || in_array($tokens[$prevIndex]->getId(), $wrongTokens, true)) {
             continue;
         }
         $tokens[$index]->setContent('PHP_SAPI');
         for ($i = $openingBrace; $i <= $closingBrace; ++$i) {
             $tokens[$i]->clear();
         }
     }
 }

/**
  * {@inheritdoc}
  */
 public function fix(\SplFileInfo $file, Tokens $tokens)
 {
     $tokensAnalyzer = new TokensAnalyzer($tokens);
     foreach ($tokensAnalyzer->getImportUseIndexes() as $index) {
         $indent = '';
         // if previous line ends with comment and current line starts with whitespace, use current indent
         if ($tokens[$index - 1]->isWhitespace(" \t") && $tokens[$index - 2]->isGivenKind(T_COMMENT)) {
             $indent = $tokens[$index - 1]->getContent();
         } elseif ($tokens[$index - 1]->isWhitespace()) {
             $indent = Utils::calculateTrailingWhitespaceIndent($tokens[$index - 1]);
         }
         $newline = "\n";
         // Handle insert index for inline T_COMMENT with whitespace after semicolon
         $semicolonIndex = $tokens->getNextTokenOfKind($index, array(';', '{'));
         $insertIndex = $semicolonIndex + 1;
         if ($tokens[$insertIndex]->isWhitespace(" \t") && $tokens[$insertIndex + 1]->isComment()) {
             ++$insertIndex;
         }
         // Increment insert index for inline T_COMMENT or T_DOC_COMMENT
         if ($tokens[$insertIndex]->isComment()) {
             ++$insertIndex;
         }
         $afterSemicolon = $tokens->getNextMeaningfulToken($semicolonIndex);
         if (!$tokens[$afterSemicolon]->isGivenKind(T_USE)) {
             $newline .= "\n";
         }
         if ($tokens[$insertIndex]->isWhitespace()) {
             $nextToken = $tokens[$insertIndex];
             $nextToken->setContent($newline . $indent . ltrim($nextToken->getContent()));
         } elseif ($newline && $indent) {
             $tokens->insertAt($insertIndex, new Token(array(T_WHITESPACE, $newline . $indent)));
         }
     }
 }

/**
  * Clear comment token, but preserve trailing linebreak if there is any.
  *
  * @param Tokens $tokens
  * @param int    $index  T_COMMENT index
  *
  * @deprecated Will be removed in the 2.0
  */
 private function clearCommentToken(Tokens $tokens, $index)
 {
     if ("\n" !== substr($tokens[$index]->getContent(), -1, 1)) {
         $tokens->clearTokenAndMergeSurroundingWhitespace($index);
         return;
     }
     // if previous not-cleared token is whitespace;
     // append line break to content
     $previous = $tokens->getNonEmptySibling($index, -1);
     if ($tokens[$previous]->isWhitespace()) {
         $tokens[$previous]->setContent($tokens[$previous]->getContent() . "\n");
         $tokens->clearTokenAndMergeSurroundingWhitespace($index);
         return;
     }
     // elseif the next not-cleared token is whitespace;
     // prepend with line break
     $next = $tokens->getNonEmptySibling($index, 1);
     if (null !== $next && $tokens[$next]->isWhitespace()) {
         $tokens[$next]->setContent("\n" . $tokens[$next]->getContent());
         $tokens->clearTokenAndMergeSurroundingWhitespace($index);
         return;
     }
     // else
     // override with whitespace token linebreak
     $tokens->overrideAt($index, array(T_WHITESPACE, "\n", $tokens[$index]->getLine()));
 }