remove python test & dependency
parent
dc84d05764
commit
c33146bdb8
@ -1,144 +0,0 @@
|
|||||||
#lang brag
|
|
||||||
|
|
||||||
# Grammar for Python
|
|
||||||
|
|
||||||
# Note: Changing the grammar specified in this file will most likely
|
|
||||||
# require corresponding changes in the parser module
|
|
||||||
# (../Modules/parsermodule.c). If you can't make the changes to
|
|
||||||
# that module yourself, please co-ordinate the required changes
|
|
||||||
# with someone who can; ask around on python-dev for help. Fred
|
|
||||||
# Drake <fdrake@acm.org> will probably be listening there.
|
|
||||||
|
|
||||||
# NOTE WELL: You should also follow all the steps listed in PEP 306,
|
|
||||||
# "How to Change Python's Grammar"
|
|
||||||
|
|
||||||
# Start symbols for the grammar:
|
|
||||||
# single_input is a single interactive statement;
|
|
||||||
# file_input is a module or sequence of commands read from an input file;
|
|
||||||
# eval_input is the input for the eval() and input() functions.
|
|
||||||
# NB: compound_stmt in single_input is followed by extra NEWLINE!
|
|
||||||
file_input: (NEWLINE | stmt)* ENDMARKER
|
|
||||||
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
|
|
||||||
eval_input: testlist NEWLINE* ENDMARKER
|
|
||||||
|
|
||||||
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
|
||||||
decorators: decorator+
|
|
||||||
decorated: decorators (classdef | funcdef)
|
|
||||||
funcdef: 'def' NAME parameters ':' suite
|
|
||||||
parameters: '(' [varargslist] ')'
|
|
||||||
varargslist: ((fpdef ['=' test] ',')*
|
|
||||||
('*' NAME [',' '**' NAME] | '**' NAME) |
|
|
||||||
fpdef ['=' test] (',' fpdef ['=' test])* [','])
|
|
||||||
fpdef: NAME | '(' fplist ')'
|
|
||||||
fplist: fpdef (',' fpdef)* [',']
|
|
||||||
|
|
||||||
stmt: simple_stmt | compound_stmt
|
|
||||||
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
|
|
||||||
small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt |
|
|
||||||
import_stmt | global_stmt | exec_stmt | assert_stmt)
|
|
||||||
expr_stmt: testlist (augassign (yield_expr|testlist) |
|
|
||||||
('=' (yield_expr|testlist))*)
|
|
||||||
augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
|
|
||||||
'<<=' | '>>=' | '**=' | '//=')
|
|
||||||
# For normal assignments, additional restrictions enforced by the interpreter
|
|
||||||
print_stmt: 'print' ( [ test (',' test)* [','] ] |
|
|
||||||
'>>' test [ (',' test)+ [','] ] )
|
|
||||||
del_stmt: 'del' exprlist
|
|
||||||
pass_stmt: 'pass'
|
|
||||||
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
|
|
||||||
break_stmt: 'break'
|
|
||||||
continue_stmt: 'continue'
|
|
||||||
return_stmt: 'return' [testlist]
|
|
||||||
yield_stmt: yield_expr
|
|
||||||
raise_stmt: 'raise' [test [',' test [',' test]]]
|
|
||||||
import_stmt: import_name | import_from
|
|
||||||
import_name: 'import' dotted_as_names
|
|
||||||
import_from: ('from' ('.'* dotted_name | '.'+)
|
|
||||||
'import' ('*' | '(' import_as_names ')' | import_as_names))
|
|
||||||
import_as_name: NAME ['as' NAME]
|
|
||||||
dotted_as_name: dotted_name ['as' NAME]
|
|
||||||
import_as_names: import_as_name (',' import_as_name)* [',']
|
|
||||||
dotted_as_names: dotted_as_name (',' dotted_as_name)*
|
|
||||||
dotted_name: NAME ('.' NAME)*
|
|
||||||
global_stmt: 'global' NAME (',' NAME)*
|
|
||||||
exec_stmt: 'exec' expr ['in' test [',' test]]
|
|
||||||
assert_stmt: 'assert' test [',' test]
|
|
||||||
|
|
||||||
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
|
|
||||||
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
|
|
||||||
while_stmt: 'while' test ':' suite ['else' ':' suite]
|
|
||||||
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
|
|
||||||
try_stmt: ('try' ':' suite
|
|
||||||
((except_clause ':' suite)+
|
|
||||||
['else' ':' suite]
|
|
||||||
['finally' ':' suite] |
|
|
||||||
'finally' ':' suite))
|
|
||||||
with_stmt: 'with' with_item (',' with_item)* ':' suite
|
|
||||||
with_item: test ['as' expr]
|
|
||||||
# NB compile.c makes sure that the default except clause is last
|
|
||||||
except_clause: 'except' [test [('as' | ',') test]]
|
|
||||||
suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
|
|
||||||
|
|
||||||
# Backward compatibility cruft to support:
|
|
||||||
# [ x for x in lambda: True, lambda: False if x() ]
|
|
||||||
# even while also allowing:
|
|
||||||
# lambda x: 5 if x else 2
|
|
||||||
# (But not a mix of the two)
|
|
||||||
testlist_safe: old_test [(',' old_test)+ [',']]
|
|
||||||
old_test: or_test | old_lambdef
|
|
||||||
old_lambdef: 'lambda' [varargslist] ':' old_test
|
|
||||||
|
|
||||||
test: or_test ['if' or_test 'else' test] | lambdef
|
|
||||||
or_test: and_test ('or' and_test)*
|
|
||||||
and_test: not_test ('and' not_test)*
|
|
||||||
not_test: 'not' not_test | comparison
|
|
||||||
comparison: expr (comp_op expr)*
|
|
||||||
comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
|
|
||||||
expr: xor_expr ('|' xor_expr)*
|
|
||||||
xor_expr: and_expr ('^' and_expr)*
|
|
||||||
and_expr: shift_expr ('&' shift_expr)*
|
|
||||||
shift_expr: arith_expr (('<<'|'>>') arith_expr)*
|
|
||||||
arith_expr: term (('+'|'-') term)*
|
|
||||||
term: factor (('*'|'/'|'%'|'//') factor)*
|
|
||||||
factor: ('+'|'-'|'~') factor | power
|
|
||||||
power: atom trailer* ['**' factor]
|
|
||||||
atom: ('(' [yield_expr|testlist_comp] ')' |
|
|
||||||
'[' [listmaker] ']' |
|
|
||||||
'{' [dictorsetmaker] '}' |
|
|
||||||
'`' testlist1 '`' |
|
|
||||||
NAME | NUMBER | STRING+)
|
|
||||||
listmaker: test ( list_for | (',' test)* [','] )
|
|
||||||
testlist_comp: test ( comp_for | (',' test)* [','] )
|
|
||||||
lambdef: 'lambda' [varargslist] ':' test
|
|
||||||
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
|
|
||||||
subscriptlist: subscript (',' subscript)* [',']
|
|
||||||
subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
|
|
||||||
sliceop: ':' [test]
|
|
||||||
exprlist: expr (',' expr)* [',']
|
|
||||||
testlist: test (',' test)* [',']
|
|
||||||
dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
|
|
||||||
(test (comp_for | (',' test)* [','])) )
|
|
||||||
|
|
||||||
classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
|
|
||||||
|
|
||||||
arglist: (argument ',')* (argument [',']
|
|
||||||
|'*' test (',' argument)* [',' '**' test]
|
|
||||||
|'**' test)
|
|
||||||
# The reason that keywords are test nodes instead of NAME is that using NAME
|
|
||||||
# results in an ambiguity. ast.c makes sure it's a NAME.
|
|
||||||
argument: test [comp_for] | test '=' test
|
|
||||||
|
|
||||||
list_iter: list_for | list_if
|
|
||||||
list_for: 'for' exprlist 'in' testlist_safe [list_iter]
|
|
||||||
list_if: 'if' old_test [list_iter]
|
|
||||||
|
|
||||||
comp_iter: comp_for | comp_if
|
|
||||||
comp_for: 'for' exprlist 'in' or_test [comp_iter]
|
|
||||||
comp_if: 'if' old_test [comp_iter]
|
|
||||||
|
|
||||||
testlist1: test (',' test)*
|
|
||||||
|
|
||||||
# not used in grammar, but may appear in "node" passed from Parser to Compiler
|
|
||||||
encoding_decl: NAME
|
|
||||||
|
|
||||||
yield_expr: 'yield' [testlist]
|
|
||||||
@ -1,136 +0,0 @@
|
|||||||
#lang racket
|
|
||||||
(require brag/examples/python-grammar
|
|
||||||
brag/support
|
|
||||||
python-tokenizer
|
|
||||||
racket/generator
|
|
||||||
parser-tools/lex
|
|
||||||
racket/match
|
|
||||||
rackunit)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
(define (kludge-nl-dedent-endmarker toks)
|
|
||||||
;; Kludge! If the last three tokens in the stream are:
|
|
||||||
;; NL, DEDENT, ENDMARKER,
|
|
||||||
;; then switch them around to:
|
|
||||||
;; DEDENT, NEWLINE, ENDMARKER
|
|
||||||
;; The Python tokenizer is doing something funny here, and I think
|
|
||||||
;; it's a bug in tokenize.py (and, transitively, the python-tokenizer
|
|
||||||
;; PLaneT package).
|
|
||||||
(cond [(< (length toks) 3)
|
|
||||||
toks]
|
|
||||||
[else
|
|
||||||
(define last-three-toks (take-right toks 3))
|
|
||||||
(match last-three-toks
|
|
||||||
[(list (list 'NL nl-text start-loc end-loc rest-str)
|
|
||||||
(and t2 (list 'DEDENT _ ...))
|
|
||||||
(and t3 (list 'ENDMARKER _ ...)))
|
|
||||||
(append (drop-right toks 3)
|
|
||||||
(list t2
|
|
||||||
(list 'NEWLINE nl-text start-loc end-loc rest-str)
|
|
||||||
t3))]
|
|
||||||
[else
|
|
||||||
toks])]))
|
|
||||||
|
|
||||||
(define (adapt-python-tokenizer ip #:end-marker-to-eof? [end-marker-to-eof? #f])
|
|
||||||
(define generated-tokens (kludge-nl-dedent-endmarker
|
|
||||||
(sequence->list (generate-tokens ip))))
|
|
||||||
(define tokens (sequence->generator generated-tokens))
|
|
||||||
(lambda ()
|
|
||||||
(let loop ()
|
|
||||||
(define next-token (tokens))
|
|
||||||
(match next-token
|
|
||||||
[(list type text (list start-line start-col) (list end-line end-col) rest-string)
|
|
||||||
;; FIXME: improve the Python tokenizer to hold offsets too.
|
|
||||||
(define start-pos (position #f start-line start-col))
|
|
||||||
(define end-pos (position #f end-line end-col))
|
|
||||||
(cond
|
|
||||||
[(eq? type 'NL)
|
|
||||||
;; Skip over NL tokens: they are meant to represent the continuation
|
|
||||||
;; of a logical line.
|
|
||||||
(loop)]
|
|
||||||
[else
|
|
||||||
(position-token (case type
|
|
||||||
[(NAME)
|
|
||||||
(cond [(set-member? all-token-types (string->symbol text))
|
|
||||||
(token (string->symbol text) text)]
|
|
||||||
[else
|
|
||||||
(token 'NAME text)])]
|
|
||||||
[(OP)
|
|
||||||
(token (string->symbol text) text)]
|
|
||||||
[(NUMBER)
|
|
||||||
(token 'NUMBER text)]
|
|
||||||
[(STRING)
|
|
||||||
(token 'STRING text)]
|
|
||||||
[(COMMENT)
|
|
||||||
(token 'WHITESPACE #:skip? #t)]
|
|
||||||
[(NEWLINE)
|
|
||||||
(token 'NEWLINE text)]
|
|
||||||
[(DEDENT)
|
|
||||||
(token 'DEDENT text)]
|
|
||||||
[(INDENT)
|
|
||||||
(token 'INDENT text)]
|
|
||||||
[(ERRORTOKEN)
|
|
||||||
(error 'uh-oh)]
|
|
||||||
[(ENDMARKER)
|
|
||||||
(if end-marker-to-eof?
|
|
||||||
(token eof)
|
|
||||||
(token 'ENDMARKER text))])
|
|
||||||
start-pos
|
|
||||||
end-pos)])]
|
|
||||||
[(? void)
|
|
||||||
(token eof)]))))
|
|
||||||
|
|
||||||
|
|
||||||
(define sample-tokens (adapt-python-tokenizer
|
|
||||||
(open-input-string #<<EOF
|
|
||||||
import blah
|
|
||||||
def hello(x):
|
|
||||||
print "hello", repr(x)
|
|
||||||
blah.baz()
|
|
||||||
|
|
||||||
EOF
|
|
||||||
)))
|
|
||||||
|
|
||||||
|
|
||||||
(void #;pretty-write
|
|
||||||
(syntax->datum (parse "hello.py" sample-tokens)))
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
(define parse-expr (make-rule-parser expr))
|
|
||||||
|
|
||||||
|
|
||||||
(check-equal?
|
|
||||||
(syntax->datum (parse-expr
|
|
||||||
(adapt-python-tokenizer (open-input-string "42")
|
|
||||||
#:end-marker-to-eof? #t)))
|
|
||||||
'(expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "42"))))))))))
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
(check-equal?
|
|
||||||
(syntax->datum (parse-expr
|
|
||||||
(adapt-python-tokenizer (open-input-string "(lambda x,y: y,x)")
|
|
||||||
#:end-marker-to-eof? #t)))
|
|
||||||
'(expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "(" (testlist_comp (test (lambdef "lambda" (varargslist (fpdef "x") "," (fpdef "y")) ":" (test (or_test (and_test (not_test (comparison (expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "y")))))))))))))))) "," (test (or_test (and_test (not_test (comparison (expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "x"))))))))))))))) ")"))))))))))
|
|
||||||
|
|
||||||
|
|
||||||
(check-equal?
|
|
||||||
(syntax->datum (parse-expr
|
|
||||||
(adapt-python-tokenizer (open-input-string "sqrt(x^2+y^2)")
|
|
||||||
#:end-marker-to-eof? #t)))
|
|
||||||
'(expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "sqrt") (trailer "(" (arglist (argument (test (or_test (and_test (not_test (comparison (expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "x"))))))) "^" (and_expr (shift_expr (arith_expr (term (factor (power (atom "2")))) "+" (term (factor (power (atom "y"))))))) "^" (and_expr (shift_expr (arith_expr (term (factor (power (atom "2")))))))))))))))) ")"))))))))))
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
(define parse-single-input (make-rule-parser single_input))
|
|
||||||
(check-equal?
|
|
||||||
(syntax->datum
|
|
||||||
(parse-single-input
|
|
||||||
(adapt-python-tokenizer (open-input-string "def f(x):\n return x*x\n\n")
|
|
||||||
#:end-marker-to-eof? #t)))
|
|
||||||
'(single_input
|
|
||||||
(compound_stmt
|
|
||||||
(funcdef "def" "f" (parameters "(" (varargslist (fpdef "x")) ")") ":" (suite "\n" " " (stmt (simple_stmt (small_stmt (flow_stmt (return_stmt "return" (testlist (test (or_test (and_test (not_test (comparison (expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "x"))) "*" (factor (power (atom "x")))))))))))))))))) "\n")) ""))) "\n"))
|
|
||||||
Loading…
Reference in New Issue