tools/closure_linter/closure_linter/indentation.py
#!/usr/bin/env python
#
# Copyright 2010 The Closure Linter Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Methods for checking EcmaScript files for indentation issues."""
__author__ = ('robbyw@google.com (Robert Walker)')
from closure_linter import ecmametadatapass
from closure_linter import errors
from closure_linter import javascripttokens
from closure_linter import tokenutil
from closure_linter.common import error
from closure_linter.common import position
import gflags as flags
flags.DEFINE_boolean('debug_indentation', False,
'Whether to print debugging information for indentation.')
# Shorthand
Context = ecmametadatapass.EcmaContext
Error = error.Error
Position = position.Position
Type = javascripttokens.JavaScriptTokenType
# The general approach:
#
# 1. Build a stack of tokens that can affect indentation.
# For each token, we determine if it is a block or continuation token.
# Some tokens need to be temporarily overwritten in case they are removed
# before the end of the line.
# Much of the work here is determining which tokens to keep on the stack
# at each point. Operators, for example, should be removed once their
# expression or line is gone, while parentheses must stay until the matching
# end parentheses is found.
#
# 2. Given that stack, determine the allowable indentations.
# Due to flexible indentation rules in JavaScript, there may be many
# allowable indentations for each stack. We follows the general
# "no false positives" approach of GJsLint and build the most permissive
# set possible.
class TokenInfo(object):
"""Stores information about a token.
Attributes:
token: The token
is_block: Whether the token represents a block indentation.
is_transient: Whether the token should be automatically removed without
finding a matching end token.
overridden_by: TokenInfo for a token that overrides the indentation that
this token would require.
is_permanent_override: Whether the override on this token should persist
even after the overriding token is removed from the stack. For example:
x([
1],
2);
needs this to be set so the last line is not required to be a continuation
indent.
line_number: The effective line number of this token. Will either be the
actual line number or the one before it in the case of a mis-wrapped
operator.
"""
def __init__(self, token, is_block=False):
"""Initializes a TokenInfo object.
Args:
token: The token
is_block: Whether the token represents a block indentation.
"""
self.token = token
self.overridden_by = None
self.is_permanent_override = False
self.is_block = is_block
self.is_transient = not is_block and not token.type in (
Type.START_PAREN, Type.START_PARAMETERS)
self.line_number = token.line_number
def __repr__(self):
result = '\n %s' % self.token
if self.overridden_by:
result = '%s OVERRIDDEN [by "%s"]' % (
result, self.overridden_by.token.string)
result += ' {is_block: %s, is_transient: %s}' % (
self.is_block, self.is_transient)
return result
class IndentationRules(object):
"""EmcaScript indentation rules.
Can be used to find common indentation errors in JavaScript, ActionScript and
other Ecma like scripting languages.
"""
def __init__(self):
"""Initializes the IndentationRules checker."""
self._stack = []
# Map from line number to number of characters it is off in indentation.
self._start_index_offset = {}
def Finalize(self):
if self._stack:
old_stack = self._stack
self._stack = []
raise Exception("INTERNAL ERROR: indentation stack is not empty: %r" %
old_stack)
def CheckToken(self, token, state):
"""Checks a token for indentation errors.
Args:
token: The current token under consideration
state: Additional information about the current tree state
Returns:
An error array [error code, error string, error token] if the token is
improperly indented, or None if indentation is correct.
"""
token_type = token.type
indentation_errors = []
stack = self._stack
is_first = self._IsFirstNonWhitespaceTokenInLine(token)
# Add tokens that could decrease indentation before checking.
if token_type == Type.END_PAREN:
self._PopTo(Type.START_PAREN)
elif token_type == Type.END_PARAMETERS:
self._PopTo(Type.START_PARAMETERS)
elif token_type == Type.END_BRACKET:
self._PopTo(Type.START_BRACKET)
elif token_type == Type.END_BLOCK:
self._PopTo(Type.START_BLOCK)
elif token_type == Type.KEYWORD and token.string in ('case', 'default'):
self._Add(self._PopTo(Type.START_BLOCK))
elif is_first and token.string == '.':
# This token should have been on the previous line, so treat it as if it
# was there.
info = TokenInfo(token)
info.line_number = token.line_number - 1
self._Add(info)
elif token_type == Type.SEMICOLON:
self._PopTransient()
not_binary_operator = (token_type != Type.OPERATOR or
token.metadata.IsUnaryOperator())
not_dot = token.string != '.'
if is_first and not_binary_operator and not_dot and token.type not in (
Type.COMMENT, Type.DOC_PREFIX, Type.STRING_TEXT):
if flags.FLAGS.debug_indentation:
print 'Line #%d: stack %r' % (token.line_number, stack)
# Ignore lines that start in JsDoc since we don't check them properly yet.
# TODO(robbyw): Support checking JsDoc indentation.
# Ignore lines that start as multi-line strings since indentation is N/A.
# Ignore lines that start with operators since we report that already.
# Ignore lines with tabs since we report that already.
expected = self._GetAllowableIndentations()
actual = self._GetActualIndentation(token)
# Special case comments describing else, case, and default. Allow them
# to outdent to the parent block.
if token_type in Type.COMMENT_TYPES:
next_code = tokenutil.SearchExcept(token, Type.NON_CODE_TYPES)
if next_code and next_code.type == Type.END_BLOCK:
next_code = tokenutil.SearchExcept(next_code, Type.NON_CODE_TYPES)
if next_code and next_code.string in ('else', 'case', 'default'):
# TODO(robbyw): This almost certainly introduces false negatives.
expected |= self._AddToEach(expected, -2)
if actual >= 0 and actual not in expected:
expected = sorted(expected)
indentation_errors.append([
errors.WRONG_INDENTATION,
'Wrong indentation: expected any of {%s} but got %d' % (
', '.join(
['%d' % x for x in expected]), actual),
token,
Position(actual, expected[0])])
self._start_index_offset[token.line_number] = expected[0] - actual
# Add tokens that could increase indentation.
if token_type == Type.START_BRACKET:
self._Add(TokenInfo(token=token,
is_block=token.metadata.context.type == Context.ARRAY_LITERAL))
elif token_type == Type.START_BLOCK or token.metadata.is_implied_block:
self._Add(TokenInfo(token=token, is_block=True))
elif token_type in (Type.START_PAREN, Type.START_PARAMETERS):
self._Add(TokenInfo(token=token, is_block=False))
elif token_type == Type.KEYWORD and token.string == 'return':
self._Add(TokenInfo(token))
elif not token.IsLastInLine() and (
token.IsAssignment() or token.IsOperator('?')):
self._Add(TokenInfo(token=token))
# Handle implied block closes.
if token.metadata.is_implied_block_close:
self._PopToImpliedBlock()
# Add some tokens only if they appear at the end of the line.
is_last = self._IsLastCodeInLine(token)
if is_last:
if token_type == Type.OPERATOR:
if token.string == ':':
if (stack and stack[-1].token.string == '?'):
# When a ternary : is on a different line than its '?', it doesn't
# add indentation.
if (token.line_number == stack[-1].token.line_number):
self._Add(TokenInfo(token))
elif token.metadata.context.type == Context.CASE_BLOCK:
# Pop transient tokens from say, line continuations, e.g.,
# case x.
# y:
# Want to pop the transient 4 space continuation indent.
self._PopTransient()
# Starting the body of the case statement, which is a type of
# block.
self._Add(TokenInfo(token=token, is_block=True))
elif token.metadata.context.type == Context.LITERAL_ELEMENT:
# When in an object literal, acts as operator indicating line
# continuations.
self._Add(TokenInfo(token))
pass
else:
# ':' might also be a statement label, no effect on indentation in
# this case.
pass
elif token.string != ',':
self._Add(TokenInfo(token))
else:
# The token is a comma.
if token.metadata.context.type == Context.VAR:
self._Add(TokenInfo(token))
elif token.metadata.context.type != Context.PARAMETERS:
self._PopTransient()
elif (token.string.endswith('.')
and token_type in (Type.IDENTIFIER, Type.NORMAL)):
self._Add(TokenInfo(token))
elif token_type == Type.PARAMETERS and token.string.endswith(','):
# Parameter lists.
self._Add(TokenInfo(token))
elif token.metadata.is_implied_semicolon:
self._PopTransient()
elif token.IsAssignment():
self._Add(TokenInfo(token))
return indentation_errors
def _AddToEach(self, original, amount):
"""Returns a new set with the given amount added to each element.
Args:
original: The original set of numbers
amount: The amount to add to each element
Returns:
A new set containing each element of the original set added to the amount.
"""
return set([x + amount for x in original])
_HARD_STOP_TYPES = (Type.START_PAREN, Type.START_PARAMETERS,
Type.START_BRACKET)
_HARD_STOP_STRINGS = ('return', '?')
def _IsHardStop(self, token):
"""Determines if the given token can have a hard stop after it.
Hard stops are indentations defined by the position of another token as in
indentation lined up with return, (, [, and ?.
"""
return (token.type in self._HARD_STOP_TYPES or
token.string in self._HARD_STOP_STRINGS or
token.IsAssignment())
def _GetAllowableIndentations(self):
"""Computes the set of allowable indentations.
Returns:
The set of allowable indentations, given the current stack.
"""
expected = set([0])
hard_stops = set([])
# Whether the tokens are still in the same continuation, meaning additional
# indentation is optional. As an example:
# x = 5 +
# 6 +
# 7;
# The second '+' does not add any required indentation.
in_same_continuation = False
for token_info in self._stack:
token = token_info.token
# Handle normal additive indentation tokens.
if not token_info.overridden_by and token.string != 'return':
if token_info.is_block:
expected = self._AddToEach(expected, 2)
hard_stops = self._AddToEach(hard_stops, 2)
in_same_continuation = False
elif in_same_continuation:
expected |= self._AddToEach(expected, 4)
hard_stops |= self._AddToEach(hard_stops, 4)
else:
expected = self._AddToEach(expected, 4)
hard_stops |= self._AddToEach(hard_stops, 4)
in_same_continuation = True
# Handle hard stops after (, [, return, =, and ?
if self._IsHardStop(token):
override_is_hard_stop = (token_info.overridden_by and
self._IsHardStop(token_info.overridden_by.token))
if not override_is_hard_stop:
start_index = token.start_index
if token.line_number in self._start_index_offset:
start_index += self._start_index_offset[token.line_number]
if (token.type in (Type.START_PAREN, Type.START_PARAMETERS) and
not token_info.overridden_by):
hard_stops.add(start_index + 1)
elif token.string == 'return' and not token_info.overridden_by:
hard_stops.add(start_index + 7)
elif (token.type == Type.START_BRACKET):
hard_stops.add(start_index + 1)
elif token.IsAssignment():
hard_stops.add(start_index + len(token.string) + 1)
elif token.IsOperator('?') and not token_info.overridden_by:
hard_stops.add(start_index + 2)
return (expected | hard_stops) or set([0])
def _GetActualIndentation(self, token):
"""Gets the actual indentation of the line containing the given token.
Args:
token: Any token on the line.
Returns:
The actual indentation of the line containing the given token. Returns
-1 if this line should be ignored due to the presence of tabs.
"""
# Move to the first token in the line
token = tokenutil.GetFirstTokenInSameLine(token)
# If it is whitespace, it is the indentation.
if token.type == Type.WHITESPACE:
if token.string.find('\t') >= 0:
return -1
else:
return len(token.string)
elif token.type == Type.PARAMETERS:
return len(token.string) - len(token.string.lstrip())
else:
return 0
def _IsFirstNonWhitespaceTokenInLine(self, token):
"""Determines if the given token is the first non-space token on its line.
Args:
token: The token.
Returns:
True if the token is the first non-whitespace token on its line.
"""
if token.type in (Type.WHITESPACE, Type.BLANK_LINE):
return False
if token.IsFirstInLine():
return True
return (token.previous and token.previous.IsFirstInLine() and
token.previous.type == Type.WHITESPACE)
def _IsLastCodeInLine(self, token):
"""Determines if the given token is the last code token on its line.
Args:
token: The token.
Returns:
True if the token is the last code token on its line.
"""
if token.type in Type.NON_CODE_TYPES:
return False
start_token = token
while True:
token = token.next
if not token or token.line_number != start_token.line_number:
return True
if token.type not in Type.NON_CODE_TYPES:
return False
def _Add(self, token_info):
"""Adds the given token info to the stack.
Args:
token_info: The token information to add.
"""
if self._stack and self._stack[-1].token == token_info.token:
# Don't add the same token twice.
return
if token_info.is_block or token_info.token.type == Type.START_PAREN:
index = 1
while index <= len(self._stack):
stack_info = self._stack[-index]
stack_token = stack_info.token
if stack_info.line_number == token_info.line_number:
# In general, tokens only override each other when they are on
# the same line.
stack_info.overridden_by = token_info
if (token_info.token.type == Type.START_BLOCK and
(stack_token.IsAssignment() or
stack_token.type in (Type.IDENTIFIER, Type.START_PAREN))):
# Multi-line blocks have lasting overrides, as in:
# callFn({
# a: 10
# },
# 30);
close_block = token_info.token.metadata.context.end_token
stack_info.is_permanent_override = \
close_block.line_number != token_info.token.line_number
elif (token_info.token.type == Type.START_BLOCK and
token_info.token.metadata.context.type == Context.BLOCK and
(stack_token.IsAssignment() or
stack_token.type == Type.IDENTIFIER)):
# When starting a function block, the override can transcend lines.
# For example
# long.long.name = function(
# a) {
# In this case the { and the = are on different lines. But the
# override should still apply.
stack_info.overridden_by = token_info
stack_info.is_permanent_override = True
else:
break
index += 1
self._stack.append(token_info)
def _Pop(self):
"""Pops the top token from the stack.
Returns:
The popped token info.
"""
token_info = self._stack.pop()
if token_info.token.type not in (Type.START_BLOCK, Type.START_BRACKET):
# Remove any temporary overrides.
self._RemoveOverrides(token_info)
else:
# For braces and brackets, which can be object and array literals, remove
# overrides when the literal is closed on the same line.
token_check = token_info.token
same_type = token_check.type
goal_type = None
if token_info.token.type == Type.START_BRACKET:
goal_type = Type.END_BRACKET
else:
goal_type = Type.END_BLOCK
line_number = token_info.token.line_number
count = 0
while token_check and token_check.line_number == line_number:
if token_check.type == goal_type:
count -= 1
if not count:
self._RemoveOverrides(token_info)
break
if token_check.type == same_type:
count += 1
token_check = token_check.next
return token_info
def _PopToImpliedBlock(self):
"""Pops the stack until an implied block token is found."""
while not self._Pop().token.metadata.is_implied_block:
pass
def _PopTo(self, stop_type):
"""Pops the stack until a token of the given type is popped.
Args:
stop_type: The type of token to pop to.
Returns:
The token info of the given type that was popped.
"""
last = None
while True:
last = self._Pop()
if last.token.type == stop_type:
break
return last
def _RemoveOverrides(self, token_info):
"""Marks any token that was overridden by this token as active again.
Args:
token_info: The token that is being removed from the stack.
"""
for stack_token in self._stack:
if (stack_token.overridden_by == token_info and
not stack_token.is_permanent_override):
stack_token.overridden_by = None
def _PopTransient(self):
"""Pops all transient tokens - i.e. not blocks, literals, or parens."""
while self._stack and self._stack[-1].is_transient:
self._Pop()