änderung1
This commit is contained in:
khaloufam70043 committed
1 parent
cc1d096bc3
commit
cbdda613b7
2098 files changed
+274524
No files matched your search
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
"""Module that contains the command line app.
|
||||
|
||||
Why does this file exist, and why not put this in __main__?
|
||||
You might be tempted to import things from __main__ later, but that will
|
||||
cause problems: the code will get executed twice:
|
||||
- When you run `python -m sqlparse` python will execute
|
||||
``__main__.py`` as a script. That means there won't be any
|
||||
``sqlparse.__main__`` in ``sys.modules``.
|
||||
- When you import __main__ it will get executed again (as a module) because
|
||||
there's no ``sqlparse.__main__`` in ``sys.modules``.
|
||||
Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from io import TextIOWrapper
|
||||
from codecs import open, getreader
|
||||
|
||||
import sqlparse
|
||||
from sqlparse.compat import PY2
|
||||
from sqlparse.exceptions import SQLParseError
|
||||
|
||||
|
||||
# TODO: Add CLI Tests
|
||||
# TODO: Simplify formatter by using argparse `type` arguments
|
||||
def create_parser():
|
||||
_CASE_CHOICES = ['upper', 'lower', 'capitalize']
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='sqlformat',
|
||||
description='Format FILE according to OPTIONS. Use "-" as FILE '
|
||||
'to read from stdin.',
|
||||
usage='%(prog)s [OPTIONS] FILE, ...',
|
||||
)
|
||||
|
||||
parser.add_argument('filename')
|
||||
|
||||
parser.add_argument(
|
||||
'-o', '--outfile',
|
||||
dest='outfile',
|
||||
metavar='FILE',
|
||||
help='write output to FILE (defaults to stdout)')
|
||||
|
||||
parser.add_argument(
|
||||
'--version',
|
||||
action='version',
|
||||
version=sqlparse.__version__)
|
||||
|
||||
group = parser.add_argument_group('Formatting Options')
|
||||
|
||||
group.add_argument(
|
||||
'-k', '--keywords',
|
||||
metavar='CHOICE',
|
||||
dest='keyword_case',
|
||||
choices=_CASE_CHOICES,
|
||||
help='change case of keywords, CHOICE is one of {0}'.format(
|
||||
', '.join('"{0}"'.format(x) for x in _CASE_CHOICES)))
|
||||
|
||||
group.add_argument(
|
||||
'-i', '--identifiers',
|
||||
metavar='CHOICE',
|
||||
dest='identifier_case',
|
||||
choices=_CASE_CHOICES,
|
||||
help='change case of identifiers, CHOICE is one of {0}'.format(
|
||||
', '.join('"{0}"'.format(x) for x in _CASE_CHOICES)))
|
||||
|
||||
group.add_argument(
|
||||
'-l', '--language',
|
||||
metavar='LANG',
|
||||
dest='output_format',
|
||||
choices=['python', 'php'],
|
||||
help='output a snippet in programming language LANG, '
|
||||
'choices are "python", "php"')
|
||||
|
||||
group.add_argument(
|
||||
'--strip-comments',
|
||||
dest='strip_comments',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='remove comments')
|
||||
|
||||
group.add_argument(
|
||||
'-r', '--reindent',
|
||||
dest='reindent',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='reindent statements')
|
||||
|
||||
group.add_argument(
|
||||
'--indent_width',
|
||||
dest='indent_width',
|
||||
default=2,
|
||||
type=int,
|
||||
help='indentation width (defaults to 2 spaces)')
|
||||
|
||||
group.add_argument(
|
||||
'--indent_after_first',
|
||||
dest='indent_after_first',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='indent after first line of statement (e.g. SELECT)')
|
||||
|
||||
group.add_argument(
|
||||
'--indent_columns',
|
||||
dest='indent_columns',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='indent all columns by indent_width instead of keyword length')
|
||||
|
||||
group.add_argument(
|
||||
'-a', '--reindent_aligned',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='reindent statements to aligned format')
|
||||
|
||||
group.add_argument(
|
||||
'-s', '--use_space_around_operators',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='place spaces around mathematical operators')
|
||||
|
||||
group.add_argument(
|
||||
'--wrap_after',
|
||||
dest='wrap_after',
|
||||
default=0,
|
||||
type=int,
|
||||
help='Column after which lists should be wrapped')
|
||||
|
||||
group.add_argument(
|
||||
'--comma_first',
|
||||
dest='comma_first',
|
||||
default=False,
|
||||
type=bool,
|
||||
help='Insert linebreak before comma (default False)')
|
||||
|
||||
group.add_argument(
|
||||
'--encoding',
|
||||
dest='encoding',
|
||||
default='utf-8',
|
||||
help='Specify the input encoding (default utf-8)')
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def _error(msg):
|
||||
"""Print msg and optionally exit with return code exit_."""
|
||||
sys.stderr.write(u'[ERROR] {0}\n'.format(msg))
|
||||
return 1
|
||||
|
||||
|
||||
def main(args=None):
|
||||
parser = create_parser()
|
||||
args = parser.parse_args(args)
|
||||
|
||||
if args.filename == '-': # read from stdin
|
||||
if PY2:
|
||||
data = getreader(args.encoding)(sys.stdin).read()
|
||||
else:
|
||||
wrapper = TextIOWrapper(sys.stdin.buffer, encoding=args.encoding)
|
||||
try:
|
||||
data = wrapper.read()
|
||||
finally:
|
||||
wrapper.detach()
|
||||
else:
|
||||
try:
|
||||
with open(args.filename, 'r', args.encoding) as f:
|
||||
data = ''.join(f.readlines())
|
||||
except IOError as e:
|
||||
return _error(
|
||||
u'Failed to read {0}: {1}'.format(args.filename, e))
|
||||
|
||||
close_stream = False
|
||||
if args.outfile:
|
||||
try:
|
||||
stream = open(args.outfile, 'w', args.encoding)
|
||||
close_stream = True
|
||||
except IOError as e:
|
||||
return _error(u'Failed to open {0}: {1}'.format(args.outfile, e))
|
||||
else:
|
||||
stream = sys.stdout
|
||||
|
||||
formatter_opts = vars(args)
|
||||
try:
|
||||
formatter_opts = sqlparse.formatter.validate_options(formatter_opts)
|
||||
except SQLParseError as e:
|
||||
return _error(u'Invalid options: {0}'.format(e))
|
||||
|
||||
s = sqlparse.format(data, **formatter_opts)
|
||||
stream.write(s)
|
||||
stream.flush()
|
||||
if close_stream:
|
||||
stream.close()
|
||||
return 0
|
||||
@@ -0,0 +1,45 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
"""Python 2/3 compatibility.
|
||||
|
||||
This module only exists to avoid a dependency on six
|
||||
for very trivial stuff. We only need to take care of
|
||||
string types, buffers and metaclasses.
|
||||
|
||||
Parts of the code is copied directly from six:
|
||||
https://bitbucket.org/gutworth/six
|
||||
"""
|
||||
|
||||
import sys
|
||||
from io import TextIOBase
|
||||
|
||||
PY2 = sys.version_info[0] == 2
|
||||
PY3 = sys.version_info[0] == 3
|
||||
|
||||
|
||||
if PY3:
|
||||
def unicode_compatible(cls):
|
||||
return cls
|
||||
|
||||
text_type = str
|
||||
string_types = (str,)
|
||||
from io import StringIO
|
||||
file_types = (StringIO, TextIOBase)
|
||||
|
||||
|
||||
elif PY2:
|
||||
def unicode_compatible(cls):
|
||||
cls.__unicode__ = cls.__str__
|
||||
cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
|
||||
return cls
|
||||
|
||||
text_type = unicode
|
||||
string_types = (str, unicode,)
|
||||
from StringIO import StringIO
|
||||
file_types = (file, StringIO, TextIOBase)
|
||||
@@ -0,0 +1,45 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
"""filter"""
|
||||
|
||||
from sqlparse import lexer
|
||||
from sqlparse.engine import grouping
|
||||
from sqlparse.engine.statement_splitter import StatementSplitter
|
||||
|
||||
|
||||
class FilterStack(object):
|
||||
def __init__(self):
|
||||
self.preprocess = []
|
||||
self.stmtprocess = []
|
||||
self.postprocess = []
|
||||
self._grouping = False
|
||||
|
||||
def enable_grouping(self):
|
||||
self._grouping = True
|
||||
|
||||
def run(self, sql, encoding=None):
|
||||
stream = lexer.tokenize(sql, encoding)
|
||||
# Process token stream
|
||||
for filter_ in self.preprocess:
|
||||
stream = filter_.process(stream)
|
||||
|
||||
stream = StatementSplitter().process(stream)
|
||||
|
||||
# Output: Stream processed Statements
|
||||
for stmt in stream:
|
||||
if self._grouping:
|
||||
stmt = grouping.group(stmt)
|
||||
|
||||
for filter_ in self.stmtprocess:
|
||||
filter_.process(stmt)
|
||||
|
||||
for filter_ in self.postprocess:
|
||||
stmt = filter_.process(stmt)
|
||||
|
||||
yield stmt
|
||||
@@ -0,0 +1,106 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
from sqlparse import sql, tokens as T
|
||||
|
||||
|
||||
class StatementSplitter(object):
|
||||
"""Filter that split stream at individual statements"""
|
||||
|
||||
def __init__(self):
|
||||
self._reset()
|
||||
|
||||
def _reset(self):
|
||||
"""Set the filter attributes to its default values"""
|
||||
self._in_declare = False
|
||||
self._is_create = False
|
||||
self._begin_depth = 0
|
||||
|
||||
self.consume_ws = False
|
||||
self.tokens = []
|
||||
self.level = 0
|
||||
|
||||
def _change_splitlevel(self, ttype, value):
|
||||
"""Get the new split level (increase, decrease or remain equal)"""
|
||||
# ANSI
|
||||
# if normal token return
|
||||
# wouldn't parenthesis increase/decrease a level?
|
||||
# no, inside a parenthesis can't start new statement
|
||||
if ttype not in T.Keyword:
|
||||
return 0
|
||||
|
||||
# Everything after here is ttype = T.Keyword
|
||||
# Also to note, once entered an If statement you are done and basically
|
||||
# returning
|
||||
unified = value.upper()
|
||||
|
||||
# three keywords begin with CREATE, but only one of them is DDL
|
||||
# DDL Create though can contain more words such as "or replace"
|
||||
if ttype is T.Keyword.DDL and unified.startswith('CREATE'):
|
||||
self._is_create = True
|
||||
return 0
|
||||
|
||||
# can have nested declare inside of being...
|
||||
if unified == 'DECLARE' and self._is_create and self._begin_depth == 0:
|
||||
self._in_declare = True
|
||||
return 1
|
||||
|
||||
if unified == 'BEGIN':
|
||||
self._begin_depth += 1
|
||||
if self._is_create:
|
||||
# FIXME(andi): This makes no sense.
|
||||
return 1
|
||||
return 0
|
||||
|
||||
# Should this respect a preceding BEGIN?
|
||||
# In CASE ... WHEN ... END this results in a split level -1.
|
||||
# Would having multiple CASE WHEN END and a Assignment Operator
|
||||
# cause the statement to cut off prematurely?
|
||||
if unified == 'END':
|
||||
self._begin_depth = max(0, self._begin_depth - 1)
|
||||
return -1
|
||||
|
||||
if (unified in ('IF', 'FOR', 'WHILE')
|
||||
and self._is_create and self._begin_depth > 0):
|
||||
return 1
|
||||
|
||||
if unified in ('END IF', 'END FOR', 'END WHILE'):
|
||||
return -1
|
||||
|
||||
# Default
|
||||
return 0
|
||||
|
||||
def process(self, stream):
|
||||
"""Process the stream"""
|
||||
EOS_TTYPE = T.Whitespace, T.Comment.Single
|
||||
|
||||
# Run over all stream tokens
|
||||
for ttype, value in stream:
|
||||
# Yield token if we finished a statement and there's no whitespaces
|
||||
# It will count newline token as a non whitespace. In this context
|
||||
# whitespace ignores newlines.
|
||||
# why don't multi line comments also count?
|
||||
if self.consume_ws and ttype not in EOS_TTYPE:
|
||||
yield sql.Statement(self.tokens)
|
||||
|
||||
# Reset filter and prepare to process next statement
|
||||
self._reset()
|
||||
|
||||
# Change current split level (increase, decrease or remain equal)
|
||||
self.level += self._change_splitlevel(ttype, value)
|
||||
|
||||
# Append the token to the current statement
|
||||
self.tokens.append(sql.Token(ttype, value))
|
||||
|
||||
# Check if we get the end of a statement
|
||||
if self.level <= 0 and ttype is T.Punctuation and value == ';':
|
||||
self.consume_ws = True
|
||||
|
||||
# Yield pending statement (if any)
|
||||
if self.tokens:
|
||||
yield sql.Statement(self.tokens)
|
||||
@@ -0,0 +1,13 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
"""Exceptions used in this package."""
|
||||
|
||||
|
||||
class SQLParseError(Exception):
|
||||
"""Base class for exceptions in this module."""
|
||||
@@ -0,0 +1,124 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
from sqlparse import sql, tokens as T
|
||||
from sqlparse.compat import text_type
|
||||
|
||||
|
||||
class OutputFilter(object):
|
||||
varname_prefix = ''
|
||||
|
||||
def __init__(self, varname='sql'):
|
||||
self.varname = self.varname_prefix + varname
|
||||
self.count = 0
|
||||
|
||||
def _process(self, stream, varname, has_nl):
|
||||
raise NotImplementedError
|
||||
|
||||
def process(self, stmt):
|
||||
self.count += 1
|
||||
if self.count > 1:
|
||||
varname = u'{f.varname}{f.count}'.format(f=self)
|
||||
else:
|
||||
varname = self.varname
|
||||
|
||||
has_nl = len(text_type(stmt).strip().splitlines()) > 1
|
||||
stmt.tokens = self._process(stmt.tokens, varname, has_nl)
|
||||
return stmt
|
||||
|
||||
|
||||
class OutputPythonFilter(OutputFilter):
|
||||
def _process(self, stream, varname, has_nl):
|
||||
# SQL query assignation to varname
|
||||
if self.count > 1:
|
||||
yield sql.Token(T.Whitespace, '\n')
|
||||
yield sql.Token(T.Name, varname)
|
||||
yield sql.Token(T.Whitespace, ' ')
|
||||
yield sql.Token(T.Operator, '=')
|
||||
yield sql.Token(T.Whitespace, ' ')
|
||||
if has_nl:
|
||||
yield sql.Token(T.Operator, '(')
|
||||
yield sql.Token(T.Text, "'")
|
||||
|
||||
# Print the tokens on the quote
|
||||
for token in stream:
|
||||
# Token is a new line separator
|
||||
if token.is_whitespace and '\n' in token.value:
|
||||
# Close quote and add a new line
|
||||
yield sql.Token(T.Text, " '")
|
||||
yield sql.Token(T.Whitespace, '\n')
|
||||
|
||||
# Quote header on secondary lines
|
||||
yield sql.Token(T.Whitespace, ' ' * (len(varname) + 4))
|
||||
yield sql.Token(T.Text, "'")
|
||||
|
||||
# Indentation
|
||||
after_lb = token.value.split('\n', 1)[1]
|
||||
if after_lb:
|
||||
yield sql.Token(T.Whitespace, after_lb)
|
||||
continue
|
||||
|
||||
# Token has escape chars
|
||||
elif "'" in token.value:
|
||||
token.value = token.value.replace("'", "\\'")
|
||||
|
||||
# Put the token
|
||||
yield sql.Token(T.Text, token.value)
|
||||
|
||||
# Close quote
|
||||
yield sql.Token(T.Text, "'")
|
||||
if has_nl:
|
||||
yield sql.Token(T.Operator, ')')
|
||||
|
||||
|
||||
class OutputPHPFilter(OutputFilter):
|
||||
varname_prefix = '$'
|
||||
|
||||
def _process(self, stream, varname, has_nl):
|
||||
# SQL query assignation to varname (quote header)
|
||||
if self.count > 1:
|
||||
yield sql.Token(T.Whitespace, '\n')
|
||||
yield sql.Token(T.Name, varname)
|
||||
yield sql.Token(T.Whitespace, ' ')
|
||||
if has_nl:
|
||||
yield sql.Token(T.Whitespace, ' ')
|
||||
yield sql.Token(T.Operator, '=')
|
||||
yield sql.Token(T.Whitespace, ' ')
|
||||
yield sql.Token(T.Text, '"')
|
||||
|
||||
# Print the tokens on the quote
|
||||
for token in stream:
|
||||
# Token is a new line separator
|
||||
if token.is_whitespace and '\n' in token.value:
|
||||
# Close quote and add a new line
|
||||
yield sql.Token(T.Text, ' ";')
|
||||
yield sql.Token(T.Whitespace, '\n')
|
||||
|
||||
# Quote header on secondary lines
|
||||
yield sql.Token(T.Name, varname)
|
||||
yield sql.Token(T.Whitespace, ' ')
|
||||
yield sql.Token(T.Operator, '.=')
|
||||
yield sql.Token(T.Whitespace, ' ')
|
||||
yield sql.Token(T.Text, '"')
|
||||
|
||||
# Indentation
|
||||
after_lb = token.value.split('\n', 1)[1]
|
||||
if after_lb:
|
||||
yield sql.Token(T.Whitespace, after_lb)
|
||||
continue
|
||||
|
||||
# Token has escape chars
|
||||
elif '"' in token.value:
|
||||
token.value = token.value.replace('"', '\\"')
|
||||
|
||||
# Put the token
|
||||
yield sql.Token(T.Text, token.value)
|
||||
|
||||
# Close quote
|
||||
yield sql.Token(T.Text, '"')
|
||||
yield sql.Token(T.Punctuation, ';')
|
||||
@@ -0,0 +1,241 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
from sqlparse import sql, tokens as T
|
||||
from sqlparse.compat import text_type
|
||||
from sqlparse.utils import offset, indent
|
||||
|
||||
|
||||
class ReindentFilter(object):
|
||||
def __init__(self, width=2, char=' ', wrap_after=0, n='\n',
|
||||
comma_first=False, indent_after_first=False,
|
||||
indent_columns=False):
|
||||
self.n = n
|
||||
self.width = width
|
||||
self.char = char
|
||||
self.indent = 1 if indent_after_first else 0
|
||||
self.offset = 0
|
||||
self.wrap_after = wrap_after
|
||||
self.comma_first = comma_first
|
||||
self.indent_columns = indent_columns
|
||||
self._curr_stmt = None
|
||||
self._last_stmt = None
|
||||
self._last_func = None
|
||||
|
||||
def _flatten_up_to_token(self, token):
|
||||
"""Yields all tokens up to token but excluding current."""
|
||||
if token.is_group:
|
||||
token = next(token.flatten())
|
||||
|
||||
for t in self._curr_stmt.flatten():
|
||||
if t == token:
|
||||
break
|
||||
yield t
|
||||
|
||||
@property
|
||||
def leading_ws(self):
|
||||
return self.offset + self.indent * self.width
|
||||
|
||||
def _get_offset(self, token):
|
||||
raw = u''.join(map(text_type, self._flatten_up_to_token(token)))
|
||||
line = (raw or '\n').splitlines()[-1]
|
||||
# Now take current offset into account and return relative offset.
|
||||
return len(line) - len(self.char * self.leading_ws)
|
||||
|
||||
def nl(self, offset=0):
|
||||
return sql.Token(
|
||||
T.Whitespace,
|
||||
self.n + self.char * max(0, self.leading_ws + offset))
|
||||
|
||||
def _next_token(self, tlist, idx=-1):
|
||||
split_words = ('FROM', 'STRAIGHT_JOIN$', 'JOIN$', 'AND', 'OR',
|
||||
'GROUP BY', 'ORDER BY', 'UNION', 'VALUES',
|
||||
'SET', 'BETWEEN', 'EXCEPT', 'HAVING', 'LIMIT')
|
||||
m_split = T.Keyword, split_words, True
|
||||
tidx, token = tlist.token_next_by(m=m_split, idx=idx)
|
||||
|
||||
if token and token.normalized == 'BETWEEN':
|
||||
tidx, token = self._next_token(tlist, tidx)
|
||||
|
||||
if token and token.normalized == 'AND':
|
||||
tidx, token = self._next_token(tlist, tidx)
|
||||
|
||||
return tidx, token
|
||||
|
||||
def _split_kwds(self, tlist):
|
||||
tidx, token = self._next_token(tlist)
|
||||
while token:
|
||||
pidx, prev_ = tlist.token_prev(tidx, skip_ws=False)
|
||||
uprev = text_type(prev_)
|
||||
|
||||
if prev_ and prev_.is_whitespace:
|
||||
del tlist.tokens[pidx]
|
||||
tidx -= 1
|
||||
|
||||
if not (uprev.endswith('\n') or uprev.endswith('\r')):
|
||||
tlist.insert_before(tidx, self.nl())
|
||||
tidx += 1
|
||||
|
||||
tidx, token = self._next_token(tlist, tidx)
|
||||
|
||||
def _split_statements(self, tlist):
|
||||
ttypes = T.Keyword.DML, T.Keyword.DDL
|
||||
tidx, token = tlist.token_next_by(t=ttypes)
|
||||
while token:
|
||||
pidx, prev_ = tlist.token_prev(tidx, skip_ws=False)
|
||||
if prev_ and prev_.is_whitespace:
|
||||
del tlist.tokens[pidx]
|
||||
tidx -= 1
|
||||
# only break if it's not the first token
|
||||
if prev_:
|
||||
tlist.insert_before(tidx, self.nl())
|
||||
tidx += 1
|
||||
tidx, token = tlist.token_next_by(t=ttypes, idx=tidx)
|
||||
|
||||
def _process(self, tlist):
|
||||
func_name = '_process_{cls}'.format(cls=type(tlist).__name__)
|
||||
func = getattr(self, func_name.lower(), self._process_default)
|
||||
func(tlist)
|
||||
|
||||
def _process_where(self, tlist):
|
||||
tidx, token = tlist.token_next_by(m=(T.Keyword, 'WHERE'))
|
||||
# issue121, errors in statement fixed??
|
||||
tlist.insert_before(tidx, self.nl())
|
||||
|
||||
with indent(self):
|
||||
self._process_default(tlist)
|
||||
|
||||
def _process_parenthesis(self, tlist):
|
||||
ttypes = T.Keyword.DML, T.Keyword.DDL
|
||||
_, is_dml_dll = tlist.token_next_by(t=ttypes)
|
||||
fidx, first = tlist.token_next_by(m=sql.Parenthesis.M_OPEN)
|
||||
|
||||
with indent(self, 1 if is_dml_dll else 0):
|
||||
tlist.tokens.insert(0, self.nl()) if is_dml_dll else None
|
||||
with offset(self, self._get_offset(first) + 1):
|
||||
self._process_default(tlist, not is_dml_dll)
|
||||
|
||||
def _process_function(self, tlist):
|
||||
self._last_func = tlist[0]
|
||||
self._process_default(tlist)
|
||||
|
||||
def _process_identifierlist(self, tlist):
|
||||
identifiers = list(tlist.get_identifiers())
|
||||
if self.indent_columns:
|
||||
first = next(identifiers[0].flatten())
|
||||
num_offset = 1 if self.char == '\t' else self.width
|
||||
else:
|
||||
first = next(identifiers.pop(0).flatten())
|
||||
num_offset = 1 if self.char == '\t' else self._get_offset(first)
|
||||
|
||||
if not tlist.within(sql.Function) and not tlist.within(sql.Values):
|
||||
with offset(self, num_offset):
|
||||
position = 0
|
||||
for token in identifiers:
|
||||
# Add 1 for the "," separator
|
||||
position += len(token.value) + 1
|
||||
if position > (self.wrap_after - self.offset):
|
||||
adjust = 0
|
||||
if self.comma_first:
|
||||
adjust = -2
|
||||
_, comma = tlist.token_prev(
|
||||
tlist.token_index(token))
|
||||
if comma is None:
|
||||
continue
|
||||
token = comma
|
||||
tlist.insert_before(token, self.nl(offset=adjust))
|
||||
if self.comma_first:
|
||||
_, ws = tlist.token_next(
|
||||
tlist.token_index(token), skip_ws=False)
|
||||
if (ws is not None
|
||||
and ws.ttype is not T.Text.Whitespace):
|
||||
tlist.insert_after(
|
||||
token, sql.Token(T.Whitespace, ' '))
|
||||
position = 0
|
||||
else:
|
||||
# ensure whitespace
|
||||
for token in tlist:
|
||||
_, next_ws = tlist.token_next(
|
||||
tlist.token_index(token), skip_ws=False)
|
||||
if token.value == ',' and not next_ws.is_whitespace:
|
||||
tlist.insert_after(
|
||||
token, sql.Token(T.Whitespace, ' '))
|
||||
|
||||
end_at = self.offset + sum(len(i.value) + 1 for i in identifiers)
|
||||
adjusted_offset = 0
|
||||
if (self.wrap_after > 0
|
||||
and end_at > (self.wrap_after - self.offset)
|
||||
and self._last_func):
|
||||
adjusted_offset = -len(self._last_func.value) - 1
|
||||
|
||||
with offset(self, adjusted_offset), indent(self):
|
||||
if adjusted_offset < 0:
|
||||
tlist.insert_before(identifiers[0], self.nl())
|
||||
position = 0
|
||||
for token in identifiers:
|
||||
# Add 1 for the "," separator
|
||||
position += len(token.value) + 1
|
||||
if (self.wrap_after > 0
|
||||
and position > (self.wrap_after - self.offset)):
|
||||
adjust = 0
|
||||
tlist.insert_before(token, self.nl(offset=adjust))
|
||||
position = 0
|
||||
self._process_default(tlist)
|
||||
|
||||
def _process_case(self, tlist):
|
||||
iterable = iter(tlist.get_cases())
|
||||
cond, _ = next(iterable)
|
||||
first = next(cond[0].flatten())
|
||||
|
||||
with offset(self, self._get_offset(tlist[0])):
|
||||
with offset(self, self._get_offset(first)):
|
||||
for cond, value in iterable:
|
||||
token = value[0] if cond is None else cond[0]
|
||||
tlist.insert_before(token, self.nl())
|
||||
|
||||
# Line breaks on group level are done. let's add an offset of
|
||||
# len "when ", "then ", "else "
|
||||
with offset(self, len("WHEN ")):
|
||||
self._process_default(tlist)
|
||||
end_idx, end = tlist.token_next_by(m=sql.Case.M_CLOSE)
|
||||
if end_idx is not None:
|
||||
tlist.insert_before(end_idx, self.nl())
|
||||
|
||||
def _process_values(self, tlist):
|
||||
tlist.insert_before(0, self.nl())
|
||||
tidx, token = tlist.token_next_by(i=sql.Parenthesis)
|
||||
first_token = token
|
||||
while token:
|
||||
ptidx, ptoken = tlist.token_next_by(m=(T.Punctuation, ','),
|
||||
idx=tidx)
|
||||
if ptoken:
|
||||
if self.comma_first:
|
||||
adjust = -2
|
||||
offset = self._get_offset(first_token) + adjust
|
||||
tlist.insert_before(ptoken, self.nl(offset))
|
||||
else:
|
||||
tlist.insert_after(ptoken,
|
||||
self.nl(self._get_offset(token)))
|
||||
tidx, token = tlist.token_next_by(i=sql.Parenthesis, idx=tidx)
|
||||
|
||||
def _process_default(self, tlist, stmts=True):
|
||||
self._split_statements(tlist) if stmts else None
|
||||
self._split_kwds(tlist)
|
||||
for sgroup in tlist.get_sublists():
|
||||
self._process(sgroup)
|
||||
|
||||
def process(self, stmt):
|
||||
self._curr_stmt = stmt
|
||||
self._process(stmt)
|
||||
|
||||
if self._last_stmt is not None:
|
||||
nl = '\n' if text_type(self._last_stmt).endswith('\n') else '\n\n'
|
||||
stmt.tokens.insert(0, sql.Token(T.Whitespace, nl))
|
||||
|
||||
self._last_stmt = stmt
|
||||
return stmt
|
||||
@@ -0,0 +1,50 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
import re
|
||||
|
||||
from sqlparse import sql, tokens as T
|
||||
from sqlparse.compat import text_type
|
||||
|
||||
|
||||
# FIXME: Doesn't work
|
||||
class RightMarginFilter(object):
|
||||
keep_together = (
|
||||
# sql.TypeCast, sql.Identifier, sql.Alias,
|
||||
)
|
||||
|
||||
def __init__(self, width=79):
|
||||
self.width = width
|
||||
self.line = ''
|
||||
|
||||
def _process(self, group, stream):
|
||||
for token in stream:
|
||||
if token.is_whitespace and '\n' in token.value:
|
||||
if token.value.endswith('\n'):
|
||||
self.line = ''
|
||||
else:
|
||||
self.line = token.value.splitlines()[-1]
|
||||
elif token.is_group and type(token) not in self.keep_together:
|
||||
token.tokens = self._process(token, token.tokens)
|
||||
else:
|
||||
val = text_type(token)
|
||||
if len(self.line) + len(val) > self.width:
|
||||
match = re.search(r'^ +', self.line)
|
||||
if match is not None:
|
||||
indent = match.group()
|
||||
else:
|
||||
indent = ''
|
||||
yield sql.Token(T.Whitespace, '\n{0}'.format(indent))
|
||||
self.line = indent
|
||||
self.line += val
|
||||
yield token
|
||||
|
||||
def process(self, group):
|
||||
# return
|
||||
# group.tokens = self._process(group, group.tokens)
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,631 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
"""This module contains classes representing syntactical elements of SQL."""
|
||||
from __future__ import print_function
|
||||
|
||||
import re
|
||||
|
||||
from sqlparse import tokens as T
|
||||
from sqlparse.compat import string_types, text_type, unicode_compatible
|
||||
from sqlparse.utils import imt, remove_quotes
|
||||
|
||||
|
||||
@unicode_compatible
|
||||
class Token(object):
|
||||
"""Base class for all other classes in this module.
|
||||
|
||||
It represents a single token and has two instance attributes:
|
||||
``value`` is the unchanged value of the token and ``ttype`` is
|
||||
the type of the token.
|
||||
"""
|
||||
|
||||
__slots__ = ('value', 'ttype', 'parent', 'normalized', 'is_keyword',
|
||||
'is_group', 'is_whitespace')
|
||||
|
||||
def __init__(self, ttype, value):
|
||||
value = text_type(value)
|
||||
self.value = value
|
||||
self.ttype = ttype
|
||||
self.parent = None
|
||||
self.is_group = False
|
||||
self.is_keyword = ttype in T.Keyword
|
||||
self.is_whitespace = self.ttype in T.Whitespace
|
||||
self.normalized = value.upper() if self.is_keyword else value
|
||||
|
||||
def __str__(self):
|
||||
return self.value
|
||||
|
||||
# Pending tokenlist __len__ bug fix
|
||||
# def __len__(self):
|
||||
# return len(self.value)
|
||||
|
||||
def __repr__(self):
|
||||
cls = self._get_repr_name()
|
||||
value = self._get_repr_value()
|
||||
|
||||
q = u'"' if value.startswith("'") and value.endswith("'") else u"'"
|
||||
return u"<{cls} {q}{value}{q} at 0x{id:2X}>".format(
|
||||
id=id(self), **locals())
|
||||
|
||||
def _get_repr_name(self):
|
||||
return str(self.ttype).split('.')[-1]
|
||||
|
||||
def _get_repr_value(self):
|
||||
raw = text_type(self)
|
||||
if len(raw) > 7:
|
||||
raw = raw[:6] + '...'
|
||||
return re.sub(r'\s+', ' ', raw)
|
||||
|
||||
def flatten(self):
|
||||
"""Resolve subgroups."""
|
||||
yield self
|
||||
|
||||
def match(self, ttype, values, regex=False):
|
||||
"""Checks whether the token matches the given arguments.
|
||||
|
||||
*ttype* is a token type. If this token doesn't match the given token
|
||||
type.
|
||||
*values* is a list of possible values for this token. The values
|
||||
are OR'ed together so if only one of the values matches ``True``
|
||||
is returned. Except for keyword tokens the comparison is
|
||||
case-sensitive. For convenience it's OK to pass in a single string.
|
||||
If *regex* is ``True`` (default is ``False``) the given values are
|
||||
treated as regular expressions.
|
||||
"""
|
||||
type_matched = self.ttype is ttype
|
||||
if not type_matched or values is None:
|
||||
return type_matched
|
||||
|
||||
if isinstance(values, string_types):
|
||||
values = (values,)
|
||||
|
||||
if regex:
|
||||
# TODO: Add test for regex with is_keyboard = false
|
||||
flag = re.IGNORECASE if self.is_keyword else 0
|
||||
values = (re.compile(v, flag) for v in values)
|
||||
|
||||
for pattern in values:
|
||||
if pattern.search(self.normalized):
|
||||
return True
|
||||
return False
|
||||
|
||||
if self.is_keyword:
|
||||
values = (v.upper() for v in values)
|
||||
|
||||
return self.normalized in values
|
||||
|
||||
def within(self, group_cls):
|
||||
"""Returns ``True`` if this token is within *group_cls*.
|
||||
|
||||
Use this method for example to check if an identifier is within
|
||||
a function: ``t.within(sql.Function)``.
|
||||
"""
|
||||
parent = self.parent
|
||||
while parent:
|
||||
if isinstance(parent, group_cls):
|
||||
return True
|
||||
parent = parent.parent
|
||||
return False
|
||||
|
||||
def is_child_of(self, other):
|
||||
"""Returns ``True`` if this token is a direct child of *other*."""
|
||||
return self.parent == other
|
||||
|
||||
def has_ancestor(self, other):
|
||||
"""Returns ``True`` if *other* is in this tokens ancestry."""
|
||||
parent = self.parent
|
||||
while parent:
|
||||
if parent == other:
|
||||
return True
|
||||
parent = parent.parent
|
||||
return False
|
||||
|
||||
|
||||
@unicode_compatible
|
||||
class TokenList(Token):
|
||||
"""A group of tokens.
|
||||
|
||||
It has an additional instance attribute ``tokens`` which holds a
|
||||
list of child-tokens.
|
||||
"""
|
||||
|
||||
__slots__ = 'tokens'
|
||||
|
||||
def __init__(self, tokens=None):
|
||||
self.tokens = tokens or []
|
||||
[setattr(token, 'parent', self) for token in tokens]
|
||||
super(TokenList, self).__init__(None, text_type(self))
|
||||
self.is_group = True
|
||||
|
||||
def __str__(self):
|
||||
return u''.join(token.value for token in self.flatten())
|
||||
|
||||
# weird bug
|
||||
# def __len__(self):
|
||||
# return len(self.tokens)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.tokens)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.tokens[item]
|
||||
|
||||
def _get_repr_name(self):
|
||||
return type(self).__name__
|
||||
|
||||
def _pprint_tree(self, max_depth=None, depth=0, f=None, _pre=''):
|
||||
"""Pretty-print the object tree."""
|
||||
token_count = len(self.tokens)
|
||||
for idx, token in enumerate(self.tokens):
|
||||
cls = token._get_repr_name()
|
||||
value = token._get_repr_value()
|
||||
|
||||
last = idx == (token_count - 1)
|
||||
pre = u'`- ' if last else u'|- '
|
||||
|
||||
q = u'"' if value.startswith("'") and value.endswith("'") else u"'"
|
||||
print(u"{_pre}{pre}{idx} {cls} {q}{value}{q}"
|
||||
.format(**locals()), file=f)
|
||||
|
||||
if token.is_group and (max_depth is None or depth < max_depth):
|
||||
parent_pre = u' ' if last else u'| '
|
||||
token._pprint_tree(max_depth, depth + 1, f, _pre + parent_pre)
|
||||
|
||||
def get_token_at_offset(self, offset):
|
||||
"""Returns the token that is on position offset."""
|
||||
idx = 0
|
||||
for token in self.flatten():
|
||||
end = idx + len(token.value)
|
||||
if idx <= offset < end:
|
||||
return token
|
||||
idx = end
|
||||
|
||||
def flatten(self):
|
||||
"""Generator yielding ungrouped tokens.
|
||||
|
||||
This method is recursively called for all child tokens.
|
||||
"""
|
||||
for token in self.tokens:
|
||||
if token.is_group:
|
||||
for item in token.flatten():
|
||||
yield item
|
||||
else:
|
||||
yield token
|
||||
|
||||
def get_sublists(self):
|
||||
for token in self.tokens:
|
||||
if token.is_group:
|
||||
yield token
|
||||
|
||||
@property
|
||||
def _groupable_tokens(self):
|
||||
return self.tokens
|
||||
|
||||
def _token_matching(self, funcs, start=0, end=None, reverse=False):
|
||||
"""next token that match functions"""
|
||||
if start is None:
|
||||
return None
|
||||
|
||||
if not isinstance(funcs, (list, tuple)):
|
||||
funcs = (funcs,)
|
||||
|
||||
if reverse:
|
||||
assert end is None
|
||||
for idx in range(start - 2, -1, -1):
|
||||
token = self.tokens[idx]
|
||||
for func in funcs:
|
||||
if func(token):
|
||||
return idx, token
|
||||
else:
|
||||
for idx, token in enumerate(self.tokens[start:end], start=start):
|
||||
for func in funcs:
|
||||
if func(token):
|
||||
return idx, token
|
||||
return None, None
|
||||
|
||||
def token_first(self, skip_ws=True, skip_cm=False):
|
||||
"""Returns the first child token.
|
||||
|
||||
If *skip_ws* is ``True`` (the default), whitespace
|
||||
tokens are ignored.
|
||||
|
||||
if *skip_cm* is ``True`` (default: ``False``), comments are
|
||||
ignored too.
|
||||
"""
|
||||
# this on is inconsistent, using Comment instead of T.Comment...
|
||||
funcs = lambda tk: not ((skip_ws and tk.is_whitespace)
|
||||
or (skip_cm and imt(tk,
|
||||
t=T.Comment, i=Comment)))
|
||||
return self._token_matching(funcs)[1]
|
||||
|
||||
def token_next_by(self, i=None, m=None, t=None, idx=-1, end=None):
|
||||
funcs = lambda tk: imt(tk, i, m, t)
|
||||
idx += 1
|
||||
return self._token_matching(funcs, idx, end)
|
||||
|
||||
def token_not_matching(self, funcs, idx):
|
||||
funcs = (funcs,) if not isinstance(funcs, (list, tuple)) else funcs
|
||||
funcs = [lambda tk: not func(tk) for func in funcs]
|
||||
return self._token_matching(funcs, idx)
|
||||
|
||||
def token_matching(self, funcs, idx):
|
||||
return self._token_matching(funcs, idx)[1]
|
||||
|
||||
def token_prev(self, idx, skip_ws=True, skip_cm=False):
|
||||
"""Returns the previous token relative to *idx*.
|
||||
|
||||
If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
|
||||
If *skip_cm* is ``True`` comments are ignored.
|
||||
``None`` is returned if there's no previous token.
|
||||
"""
|
||||
return self.token_next(idx, skip_ws, skip_cm, _reverse=True)
|
||||
|
||||
# TODO: May need to re-add default value to idx
|
||||
def token_next(self, idx, skip_ws=True, skip_cm=False, _reverse=False):
|
||||
"""Returns the next token relative to *idx*.
|
||||
|
||||
If *skip_ws* is ``True`` (the default) whitespace tokens are ignored.
|
||||
If *skip_cm* is ``True`` comments are ignored.
|
||||
``None`` is returned if there's no next token.
|
||||
"""
|
||||
if idx is None:
|
||||
return None, None
|
||||
idx += 1 # alot of code usage current pre-compensates for this
|
||||
funcs = lambda tk: not ((skip_ws and tk.is_whitespace)
|
||||
or (skip_cm and imt(tk,
|
||||
t=T.Comment, i=Comment)))
|
||||
return self._token_matching(funcs, idx, reverse=_reverse)
|
||||
|
||||
def token_index(self, token, start=0):
|
||||
"""Return list index of token."""
|
||||
start = start if isinstance(start, int) else self.token_index(start)
|
||||
return start + self.tokens[start:].index(token)
|
||||
|
||||
def group_tokens(self, grp_cls, start, end, include_end=True,
|
||||
extend=False):
|
||||
"""Replace tokens by an instance of *grp_cls*."""
|
||||
start_idx = start
|
||||
start = self.tokens[start_idx]
|
||||
|
||||
end_idx = end + include_end
|
||||
|
||||
# will be needed later for new group_clauses
|
||||
# while skip_ws and tokens and tokens[-1].is_whitespace:
|
||||
# tokens = tokens[:-1]
|
||||
|
||||
if extend and isinstance(start, grp_cls):
|
||||
subtokens = self.tokens[start_idx + 1:end_idx]
|
||||
|
||||
grp = start
|
||||
grp.tokens.extend(subtokens)
|
||||
del self.tokens[start_idx + 1:end_idx]
|
||||
grp.value = text_type(start)
|
||||
else:
|
||||
subtokens = self.tokens[start_idx:end_idx]
|
||||
grp = grp_cls(subtokens)
|
||||
self.tokens[start_idx:end_idx] = [grp]
|
||||
grp.parent = self
|
||||
|
||||
for token in subtokens:
|
||||
token.parent = grp
|
||||
|
||||
return grp
|
||||
|
||||
def insert_before(self, where, token):
|
||||
"""Inserts *token* before *where*."""
|
||||
if not isinstance(where, int):
|
||||
where = self.token_index(where)
|
||||
token.parent = self
|
||||
self.tokens.insert(where, token)
|
||||
|
||||
def insert_after(self, where, token, skip_ws=True):
|
||||
"""Inserts *token* after *where*."""
|
||||
if not isinstance(where, int):
|
||||
where = self.token_index(where)
|
||||
nidx, next_ = self.token_next(where, skip_ws=skip_ws)
|
||||
token.parent = self
|
||||
if next_ is None:
|
||||
self.tokens.append(token)
|
||||
else:
|
||||
self.tokens.insert(nidx, token)
|
||||
|
||||
def has_alias(self):
|
||||
"""Returns ``True`` if an alias is present."""
|
||||
return self.get_alias() is not None
|
||||
|
||||
def get_alias(self):
|
||||
"""Returns the alias for this identifier or ``None``."""
|
||||
|
||||
# "name AS alias"
|
||||
kw_idx, kw = self.token_next_by(m=(T.Keyword, 'AS'))
|
||||
if kw is not None:
|
||||
return self._get_first_name(kw_idx + 1, keywords=True)
|
||||
|
||||
# "name alias" or "complicated column expression alias"
|
||||
_, ws = self.token_next_by(t=T.Whitespace)
|
||||
if len(self.tokens) > 2 and ws is not None:
|
||||
return self._get_first_name(reverse=True)
|
||||
|
||||
def get_name(self):
|
||||
"""Returns the name of this identifier.
|
||||
|
||||
This is either it's alias or it's real name. The returned valued can
|
||||
be considered as the name under which the object corresponding to
|
||||
this identifier is known within the current statement.
|
||||
"""
|
||||
return self.get_alias() or self.get_real_name()
|
||||
|
||||
def get_real_name(self):
|
||||
"""Returns the real name (object name) of this identifier."""
|
||||
# a.b
|
||||
dot_idx, _ = self.token_next_by(m=(T.Punctuation, '.'))
|
||||
return self._get_first_name(dot_idx, real_name=True)
|
||||
|
||||
def get_parent_name(self):
|
||||
"""Return name of the parent object if any.
|
||||
|
||||
A parent object is identified by the first occurring dot.
|
||||
"""
|
||||
dot_idx, _ = self.token_next_by(m=(T.Punctuation, '.'))
|
||||
_, prev_ = self.token_prev(dot_idx)
|
||||
return remove_quotes(prev_.value) if prev_ is not None else None
|
||||
|
||||
def _get_first_name(self, idx=None, reverse=False, keywords=False,
|
||||
real_name=False):
|
||||
"""Returns the name of the first token with a name"""
|
||||
|
||||
tokens = self.tokens[idx:] if idx else self.tokens
|
||||
tokens = reversed(tokens) if reverse else tokens
|
||||
types = [T.Name, T.Wildcard, T.String.Symbol]
|
||||
|
||||
if keywords:
|
||||
types.append(T.Keyword)
|
||||
|
||||
for token in tokens:
|
||||
if token.ttype in types:
|
||||
return remove_quotes(token.value)
|
||||
elif isinstance(token, (Identifier, Function)):
|
||||
return token.get_real_name() if real_name else token.get_name()
|
||||
|
||||
|
||||
class Statement(TokenList):
|
||||
"""Represents a SQL statement."""
|
||||
|
||||
def get_type(self):
|
||||
"""Returns the type of a statement.
|
||||
|
||||
The returned value is a string holding an upper-cased reprint of
|
||||
the first DML or DDL keyword. If the first token in this group
|
||||
isn't a DML or DDL keyword "UNKNOWN" is returned.
|
||||
|
||||
Whitespaces and comments at the beginning of the statement
|
||||
are ignored.
|
||||
"""
|
||||
first_token = self.token_first(skip_cm=True)
|
||||
if first_token is None:
|
||||
# An "empty" statement that either has not tokens at all
|
||||
# or only whitespace tokens.
|
||||
return 'UNKNOWN'
|
||||
|
||||
elif first_token.ttype in (T.Keyword.DML, T.Keyword.DDL):
|
||||
return first_token.normalized
|
||||
|
||||
elif first_token.ttype == T.Keyword.CTE:
|
||||
# The WITH keyword should be followed by either an Identifier or
|
||||
# an IdentifierList containing the CTE definitions; the actual
|
||||
# DML keyword (e.g. SELECT, INSERT) will follow next.
|
||||
fidx = self.token_index(first_token)
|
||||
tidx, token = self.token_next(fidx, skip_ws=True)
|
||||
if isinstance(token, (Identifier, IdentifierList)):
|
||||
_, dml_keyword = self.token_next(tidx, skip_ws=True)
|
||||
|
||||
if dml_keyword is not None \
|
||||
and dml_keyword.ttype == T.Keyword.DML:
|
||||
return dml_keyword.normalized
|
||||
|
||||
# Hmm, probably invalid syntax, so return unknown.
|
||||
return 'UNKNOWN'
|
||||
|
||||
|
||||
class Identifier(TokenList):
|
||||
"""Represents an identifier.
|
||||
|
||||
Identifiers may have aliases or typecasts.
|
||||
"""
|
||||
|
||||
def is_wildcard(self):
|
||||
"""Return ``True`` if this identifier contains a wildcard."""
|
||||
_, token = self.token_next_by(t=T.Wildcard)
|
||||
return token is not None
|
||||
|
||||
def get_typecast(self):
|
||||
"""Returns the typecast or ``None`` of this object as a string."""
|
||||
midx, marker = self.token_next_by(m=(T.Punctuation, '::'))
|
||||
nidx, next_ = self.token_next(midx, skip_ws=False)
|
||||
return next_.value if next_ else None
|
||||
|
||||
def get_ordering(self):
|
||||
"""Returns the ordering or ``None`` as uppercase string."""
|
||||
_, ordering = self.token_next_by(t=T.Keyword.Order)
|
||||
return ordering.normalized if ordering else None
|
||||
|
||||
def get_array_indices(self):
|
||||
"""Returns an iterator of index token lists"""
|
||||
|
||||
for token in self.tokens:
|
||||
if isinstance(token, SquareBrackets):
|
||||
# Use [1:-1] index to discard the square brackets
|
||||
yield token.tokens[1:-1]
|
||||
|
||||
|
||||
class IdentifierList(TokenList):
|
||||
"""A list of :class:`~sqlparse.sql.Identifier`\'s."""
|
||||
|
||||
def get_identifiers(self):
|
||||
"""Returns the identifiers.
|
||||
|
||||
Whitespaces and punctuations are not included in this generator.
|
||||
"""
|
||||
for token in self.tokens:
|
||||
if not (token.is_whitespace or token.match(T.Punctuation, ',')):
|
||||
yield token
|
||||
|
||||
|
||||
class Parenthesis(TokenList):
|
||||
"""Tokens between parenthesis."""
|
||||
M_OPEN = T.Punctuation, '('
|
||||
M_CLOSE = T.Punctuation, ')'
|
||||
|
||||
@property
|
||||
def _groupable_tokens(self):
|
||||
return self.tokens[1:-1]
|
||||
|
||||
|
||||
class SquareBrackets(TokenList):
|
||||
"""Tokens between square brackets"""
|
||||
M_OPEN = T.Punctuation, '['
|
||||
M_CLOSE = T.Punctuation, ']'
|
||||
|
||||
@property
|
||||
def _groupable_tokens(self):
|
||||
return self.tokens[1:-1]
|
||||
|
||||
|
||||
class Assignment(TokenList):
|
||||
"""An assignment like 'var := val;'"""
|
||||
|
||||
|
||||
class If(TokenList):
|
||||
"""An 'if' clause with possible 'else if' or 'else' parts."""
|
||||
M_OPEN = T.Keyword, 'IF'
|
||||
M_CLOSE = T.Keyword, 'END IF'
|
||||
|
||||
|
||||
class For(TokenList):
|
||||
"""A 'FOR' loop."""
|
||||
M_OPEN = T.Keyword, ('FOR', 'FOREACH')
|
||||
M_CLOSE = T.Keyword, 'END LOOP'
|
||||
|
||||
|
||||
class Comparison(TokenList):
|
||||
"""A comparison used for example in WHERE clauses."""
|
||||
|
||||
@property
|
||||
def left(self):
|
||||
return self.tokens[0]
|
||||
|
||||
@property
|
||||
def right(self):
|
||||
return self.tokens[-1]
|
||||
|
||||
|
||||
class Comment(TokenList):
|
||||
"""A comment."""
|
||||
|
||||
def is_multiline(self):
|
||||
return self.tokens and self.tokens[0].ttype == T.Comment.Multiline
|
||||
|
||||
|
||||
class Where(TokenList):
|
||||
"""A WHERE clause."""
|
||||
M_OPEN = T.Keyword, 'WHERE'
|
||||
M_CLOSE = T.Keyword, (
|
||||
'ORDER BY', 'GROUP BY', 'LIMIT', 'UNION', 'UNION ALL', 'EXCEPT',
|
||||
'HAVING', 'RETURNING', 'INTO')
|
||||
|
||||
|
||||
class Having(TokenList):
|
||||
"""A HAVING clause."""
|
||||
M_OPEN = T.Keyword, 'HAVING'
|
||||
M_CLOSE = T.Keyword, ('ORDER BY', 'LIMIT')
|
||||
|
||||
|
||||
class Case(TokenList):
|
||||
"""A CASE statement with one or more WHEN and possibly an ELSE part."""
|
||||
M_OPEN = T.Keyword, 'CASE'
|
||||
M_CLOSE = T.Keyword, 'END'
|
||||
|
||||
def get_cases(self, skip_ws=False):
|
||||
"""Returns a list of 2-tuples (condition, value).
|
||||
|
||||
If an ELSE exists condition is None.
|
||||
"""
|
||||
CONDITION = 1
|
||||
VALUE = 2
|
||||
|
||||
ret = []
|
||||
mode = CONDITION
|
||||
|
||||
for token in self.tokens:
|
||||
# Set mode from the current statement
|
||||
if token.match(T.Keyword, 'CASE'):
|
||||
continue
|
||||
|
||||
elif skip_ws and token.ttype in T.Whitespace:
|
||||
continue
|
||||
|
||||
elif token.match(T.Keyword, 'WHEN'):
|
||||
ret.append(([], []))
|
||||
mode = CONDITION
|
||||
|
||||
elif token.match(T.Keyword, 'THEN'):
|
||||
mode = VALUE
|
||||
|
||||
elif token.match(T.Keyword, 'ELSE'):
|
||||
ret.append((None, []))
|
||||
mode = VALUE
|
||||
|
||||
elif token.match(T.Keyword, 'END'):
|
||||
mode = None
|
||||
|
||||
# First condition without preceding WHEN
|
||||
if mode and not ret:
|
||||
ret.append(([], []))
|
||||
|
||||
# Append token depending of the current mode
|
||||
if mode == CONDITION:
|
||||
ret[-1][0].append(token)
|
||||
|
||||
elif mode == VALUE:
|
||||
ret[-1][1].append(token)
|
||||
|
||||
# Return cases list
|
||||
return ret
|
||||
|
||||
|
||||
class Function(TokenList):
|
||||
"""A function or procedure call."""
|
||||
|
||||
def get_parameters(self):
|
||||
"""Return a list of parameters."""
|
||||
parenthesis = self.tokens[-1]
|
||||
for token in parenthesis.tokens:
|
||||
if isinstance(token, IdentifierList):
|
||||
return token.get_identifiers()
|
||||
elif imt(token, i=(Function, Identifier), t=T.Literal):
|
||||
return [token, ]
|
||||
return []
|
||||
|
||||
|
||||
class Begin(TokenList):
|
||||
"""A BEGIN/END block."""
|
||||
M_OPEN = T.Keyword, 'BEGIN'
|
||||
M_CLOSE = T.Keyword, 'END'
|
||||
|
||||
|
||||
class Operation(TokenList):
|
||||
"""Grouping of operations"""
|
||||
|
||||
|
||||
class Values(TokenList):
|
||||
"""Grouping of values"""
|
||||
|
||||
|
||||
class Command(TokenList):
|
||||
"""Grouping of CLI commands."""
|
||||
@@ -0,0 +1,69 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
#
|
||||
# The Token implementation is based on pygment's token system written
|
||||
# by Georg Brandl.
|
||||
# http://pygments.org/
|
||||
|
||||
"""Tokens"""
|
||||
|
||||
|
||||
class _TokenType(tuple):
|
||||
parent = None
|
||||
|
||||
def __contains__(self, item):
|
||||
return item is not None and (self is item or item[:len(self)] == self)
|
||||
|
||||
def __getattr__(self, name):
|
||||
new = _TokenType(self + (name,))
|
||||
setattr(self, name, new)
|
||||
new.parent = self
|
||||
return new
|
||||
|
||||
def __repr__(self):
|
||||
# self can be False only if its the `root` i.e. Token itself
|
||||
return 'Token' + ('.' if self else '') + '.'.join(self)
|
||||
|
||||
|
||||
Token = _TokenType()
|
||||
|
||||
# Special token types
|
||||
Text = Token.Text
|
||||
Whitespace = Text.Whitespace
|
||||
Newline = Whitespace.Newline
|
||||
Error = Token.Error
|
||||
# Text that doesn't belong to this lexer (e.g. HTML in PHP)
|
||||
Other = Token.Other
|
||||
|
||||
# Common token types for source code
|
||||
Keyword = Token.Keyword
|
||||
Name = Token.Name
|
||||
Literal = Token.Literal
|
||||
String = Literal.String
|
||||
Number = Literal.Number
|
||||
Punctuation = Token.Punctuation
|
||||
Operator = Token.Operator
|
||||
Comparison = Operator.Comparison
|
||||
Wildcard = Token.Wildcard
|
||||
Comment = Token.Comment
|
||||
Assignment = Token.Assignment
|
||||
|
||||
# Generic types for non-source code
|
||||
Generic = Token.Generic
|
||||
Command = Generic.Command
|
||||
|
||||
# String and some others are not direct children of Token.
|
||||
# alias them:
|
||||
Token.Token = Token
|
||||
Token.String = String
|
||||
Token.Number = Number
|
||||
|
||||
# SQL specific tokens
|
||||
DML = Keyword.DML
|
||||
DDL = Keyword.DDL
|
||||
CTE = Keyword.CTE
|
||||
@@ -0,0 +1,123 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (C) 2009-2018 the sqlparse authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of python-sqlparse and is released under
|
||||
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
|
||||
|
||||
import itertools
|
||||
import re
|
||||
from collections import deque
|
||||
from contextlib import contextmanager
|
||||
from sqlparse.compat import text_type
|
||||
|
||||
# This regular expression replaces the home-cooked parser that was here before.
|
||||
# It is much faster, but requires an extra post-processing step to get the
|
||||
# desired results (that are compatible with what you would expect from the
|
||||
# str.splitlines() method).
|
||||
#
|
||||
# It matches groups of characters: newlines, quoted strings, or unquoted text,
|
||||
# and splits on that basis. The post-processing step puts those back together
|
||||
# into the actual lines of SQL.
|
||||
SPLIT_REGEX = re.compile(r"""
|
||||
(
|
||||
(?: # Start of non-capturing group
|
||||
(?:\r\n|\r|\n) | # Match any single newline, or
|
||||
[^\r\n'"]+ | # Match any character series without quotes or
|
||||
# newlines, or
|
||||
"(?:[^"\\]|\\.)*" | # Match double-quoted strings, or
|
||||
'(?:[^'\\]|\\.)*' # Match single quoted strings
|
||||
)
|
||||
)
|
||||
""", re.VERBOSE)
|
||||
|
||||
LINE_MATCH = re.compile(r'(\r\n|\r|\n)')
|
||||
|
||||
|
||||
def split_unquoted_newlines(stmt):
|
||||
"""Split a string on all unquoted newlines.
|
||||
|
||||
Unlike str.splitlines(), this will ignore CR/LF/CR+LF if the requisite
|
||||
character is inside of a string."""
|
||||
text = text_type(stmt)
|
||||
lines = SPLIT_REGEX.split(text)
|
||||
outputlines = ['']
|
||||
for line in lines:
|
||||
if not line:
|
||||
continue
|
||||
elif LINE_MATCH.match(line):
|
||||
outputlines.append('')
|
||||
else:
|
||||
outputlines[-1] += line
|
||||
return outputlines
|
||||
|
||||
|
||||
def remove_quotes(val):
|
||||
"""Helper that removes surrounding quotes from strings."""
|
||||
if val is None:
|
||||
return
|
||||
if val[0] in ('"', "'") and val[0] == val[-1]:
|
||||
val = val[1:-1]
|
||||
return val
|
||||
|
||||
|
||||
def recurse(*cls):
|
||||
"""Function decorator to help with recursion
|
||||
|
||||
:param cls: Classes to not recurse over
|
||||
:return: function
|
||||
"""
|
||||
def wrap(f):
|
||||
def wrapped_f(tlist):
|
||||
for sgroup in tlist.get_sublists():
|
||||
if not isinstance(sgroup, cls):
|
||||
wrapped_f(sgroup)
|
||||
f(tlist)
|
||||
|
||||
return wrapped_f
|
||||
|
||||
return wrap
|
||||
|
||||
|
||||
def imt(token, i=None, m=None, t=None):
|
||||
"""Helper function to simplify comparisons Instance, Match and TokenType
|
||||
:param token:
|
||||
:param i: Class or Tuple/List of Classes
|
||||
:param m: Tuple of TokenType & Value. Can be list of Tuple for multiple
|
||||
:param t: TokenType or Tuple/List of TokenTypes
|
||||
:return: bool
|
||||
"""
|
||||
clss = i
|
||||
types = [t, ] if t and not isinstance(t, list) else t
|
||||
mpatterns = [m, ] if m and not isinstance(m, list) else m
|
||||
|
||||
if token is None:
|
||||
return False
|
||||
elif clss and isinstance(token, clss):
|
||||
return True
|
||||
elif mpatterns and any(token.match(*pattern) for pattern in mpatterns):
|
||||
return True
|
||||
elif types and any(token.ttype in ttype for ttype in types):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def consume(iterator, n):
|
||||
"""Advance the iterator n-steps ahead. If n is none, consume entirely."""
|
||||
deque(itertools.islice(iterator, n), maxlen=0)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def offset(filter_, n=0):
|
||||
filter_.offset += n
|
||||
yield
|
||||
filter_.offset -= n
|
||||
|
||||
|
||||
@contextmanager
|
||||
def indent(filter_, n=1):
|
||||
filter_.indent += n
|
||||
yield
|
||||
filter_.indent -= n
|
||||
Reference in new issue
Block a user