diff --git a/brag/brag/examples/python-grammar.rkt b/brag/brag/examples/python-grammar.rkt deleted file mode 100755 index 9244845..0000000 --- a/brag/brag/examples/python-grammar.rkt +++ /dev/null @@ -1,144 +0,0 @@ -#lang brag - -# Grammar for Python - -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake will probably be listening there. - -# NOTE WELL: You should also follow all the steps listed in PEP 306, -# "How to Change Python's Grammar" - -# Start symbols for the grammar: -# single_input is a single interactive statement; -# file_input is a module or sequence of commands read from an input file; -# eval_input is the input for the eval() and input() functions. -# NB: compound_stmt in single_input is followed by extra NEWLINE! -file_input: (NEWLINE | stmt)* ENDMARKER -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE -eval_input: testlist NEWLINE* ENDMARKER - -decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE -decorators: decorator+ -decorated: decorators (classdef | funcdef) -funcdef: 'def' NAME parameters ':' suite -parameters: '(' [varargslist] ')' -varargslist: ((fpdef ['=' test] ',')* - ('*' NAME [',' '**' NAME] | '**' NAME) | - fpdef ['=' test] (',' fpdef ['=' test])* [',']) -fpdef: NAME | '(' fplist ')' -fplist: fpdef (',' fpdef)* [','] - -stmt: simple_stmt | compound_stmt -simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | - import_stmt | global_stmt | exec_stmt | assert_stmt) -expr_stmt: testlist (augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist))*) -augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | - '<<=' | '>>=' | '**=' | '//=') -# For normal assignments, additional restrictions enforced by the interpreter -print_stmt: 'print' ( [ test (',' test)* [','] ] | - '>>' test [ (',' test)+ [','] ] ) -del_stmt: 'del' exprlist -pass_stmt: 'pass' -flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt -break_stmt: 'break' -continue_stmt: 'continue' -return_stmt: 'return' [testlist] -yield_stmt: yield_expr -raise_stmt: 'raise' [test [',' test [',' test]]] -import_stmt: import_name | import_from -import_name: 'import' dotted_as_names -import_from: ('from' ('.'* dotted_name | '.'+) - 'import' ('*' | '(' import_as_names ')' | import_as_names)) -import_as_name: NAME ['as' NAME] -dotted_as_name: dotted_name ['as' NAME] -import_as_names: import_as_name (',' import_as_name)* [','] -dotted_as_names: dotted_as_name (',' dotted_as_name)* -dotted_name: NAME ('.' NAME)* -global_stmt: 'global' NAME (',' NAME)* -exec_stmt: 'exec' expr ['in' test [',' test]] -assert_stmt: 'assert' test [',' test] - -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated -if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] -while_stmt: 'while' test ':' suite ['else' ':' suite] -for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] -try_stmt: ('try' ':' suite - ((except_clause ':' suite)+ - ['else' ':' suite] - ['finally' ':' suite] | - 'finally' ':' suite)) -with_stmt: 'with' with_item (',' with_item)* ':' suite -with_item: test ['as' expr] -# NB compile.c makes sure that the default except clause is last -except_clause: 'except' [test [('as' | ',') test]] -suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT - -# Backward compatibility cruft to support: -# [ x for x in lambda: True, lambda: False if x() ] -# even while also allowing: -# lambda x: 5 if x else 2 -# (But not a mix of the two) -testlist_safe: old_test [(',' old_test)+ [',']] -old_test: or_test | old_lambdef -old_lambdef: 'lambda' [varargslist] ':' old_test - -test: or_test ['if' or_test 'else' test] | lambdef -or_test: and_test ('or' and_test)* -and_test: not_test ('and' not_test)* -not_test: 'not' not_test | comparison -comparison: expr (comp_op expr)* -comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -expr: xor_expr ('|' xor_expr)* -xor_expr: and_expr ('^' and_expr)* -and_expr: shift_expr ('&' shift_expr)* -shift_expr: arith_expr (('<<'|'>>') arith_expr)* -arith_expr: term (('+'|'-') term)* -term: factor (('*'|'/'|'%'|'//') factor)* -factor: ('+'|'-'|'~') factor | power -power: atom trailer* ['**' factor] -atom: ('(' [yield_expr|testlist_comp] ')' | - '[' [listmaker] ']' | - '{' [dictorsetmaker] '}' | - '`' testlist1 '`' | - NAME | NUMBER | STRING+) -listmaker: test ( list_for | (',' test)* [','] ) -testlist_comp: test ( comp_for | (',' test)* [','] ) -lambdef: 'lambda' [varargslist] ':' test -trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -subscriptlist: subscript (',' subscript)* [','] -subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] -sliceop: ':' [test] -exprlist: expr (',' expr)* [','] -testlist: test (',' test)* [','] -dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | - (test (comp_for | (',' test)* [','])) ) - -classdef: 'class' NAME ['(' [testlist] ')'] ':' suite - -arglist: (argument ',')* (argument [','] - |'*' test (',' argument)* [',' '**' test] - |'**' test) -# The reason that keywords are test nodes instead of NAME is that using NAME -# results in an ambiguity. ast.c makes sure it's a NAME. -argument: test [comp_for] | test '=' test - -list_iter: list_for | list_if -list_for: 'for' exprlist 'in' testlist_safe [list_iter] -list_if: 'if' old_test [list_iter] - -comp_iter: comp_for | comp_if -comp_for: 'for' exprlist 'in' or_test [comp_iter] -comp_if: 'if' old_test [comp_iter] - -testlist1: test (',' test)* - -# not used in grammar, but may appear in "node" passed from Parser to Compiler -encoding_decl: NAME - -yield_expr: 'yield' [testlist] diff --git a/brag/brag/test/exercise-python-grammar.rkt b/brag/brag/test/exercise-python-grammar.rkt deleted file mode 100755 index 21d1c85..0000000 --- a/brag/brag/test/exercise-python-grammar.rkt +++ /dev/null @@ -1,136 +0,0 @@ -#lang racket -(require brag/examples/python-grammar - brag/support - python-tokenizer - racket/generator - parser-tools/lex - racket/match - rackunit) - - - -(define (kludge-nl-dedent-endmarker toks) - ;; Kludge! If the last three tokens in the stream are: - ;; NL, DEDENT, ENDMARKER, - ;; then switch them around to: - ;; DEDENT, NEWLINE, ENDMARKER - ;; The Python tokenizer is doing something funny here, and I think - ;; it's a bug in tokenize.py (and, transitively, the python-tokenizer - ;; PLaneT package). - (cond [(< (length toks) 3) - toks] - [else - (define last-three-toks (take-right toks 3)) - (match last-three-toks - [(list (list 'NL nl-text start-loc end-loc rest-str) - (and t2 (list 'DEDENT _ ...)) - (and t3 (list 'ENDMARKER _ ...))) - (append (drop-right toks 3) - (list t2 - (list 'NEWLINE nl-text start-loc end-loc rest-str) - t3))] - [else - toks])])) - -(define (adapt-python-tokenizer ip #:end-marker-to-eof? [end-marker-to-eof? #f]) - (define generated-tokens (kludge-nl-dedent-endmarker - (sequence->list (generate-tokens ip)))) - (define tokens (sequence->generator generated-tokens)) - (lambda () - (let loop () - (define next-token (tokens)) - (match next-token - [(list type text (list start-line start-col) (list end-line end-col) rest-string) - ;; FIXME: improve the Python tokenizer to hold offsets too. - (define start-pos (position #f start-line start-col)) - (define end-pos (position #f end-line end-col)) - (cond - [(eq? type 'NL) - ;; Skip over NL tokens: they are meant to represent the continuation - ;; of a logical line. - (loop)] - [else - (position-token (case type - [(NAME) - (cond [(set-member? all-token-types (string->symbol text)) - (token (string->symbol text) text)] - [else - (token 'NAME text)])] - [(OP) - (token (string->symbol text) text)] - [(NUMBER) - (token 'NUMBER text)] - [(STRING) - (token 'STRING text)] - [(COMMENT) - (token 'WHITESPACE #:skip? #t)] - [(NEWLINE) - (token 'NEWLINE text)] - [(DEDENT) - (token 'DEDENT text)] - [(INDENT) - (token 'INDENT text)] - [(ERRORTOKEN) - (error 'uh-oh)] - [(ENDMARKER) - (if end-marker-to-eof? - (token eof) - (token 'ENDMARKER text))]) - start-pos - end-pos)])] - [(? void) - (token eof)])))) - - -(define sample-tokens (adapt-python-tokenizer - (open-input-string #<datum (parse "hello.py" sample-tokens))) - - - - -(define parse-expr (make-rule-parser expr)) - - -(check-equal? - (syntax->datum (parse-expr - (adapt-python-tokenizer (open-input-string "42") - #:end-marker-to-eof? #t))) - '(expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "42")))))))))) - - - -(check-equal? - (syntax->datum (parse-expr - (adapt-python-tokenizer (open-input-string "(lambda x,y: y,x)") - #:end-marker-to-eof? #t))) - '(expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "(" (testlist_comp (test (lambdef "lambda" (varargslist (fpdef "x") "," (fpdef "y")) ":" (test (or_test (and_test (not_test (comparison (expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "y")))))))))))))))) "," (test (or_test (and_test (not_test (comparison (expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "x"))))))))))))))) ")")))))))))) - - -(check-equal? - (syntax->datum (parse-expr - (adapt-python-tokenizer (open-input-string "sqrt(x^2+y^2)") - #:end-marker-to-eof? #t))) - '(expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "sqrt") (trailer "(" (arglist (argument (test (or_test (and_test (not_test (comparison (expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "x"))))))) "^" (and_expr (shift_expr (arith_expr (term (factor (power (atom "2")))) "+" (term (factor (power (atom "y"))))))) "^" (and_expr (shift_expr (arith_expr (term (factor (power (atom "2")))))))))))))))) ")")))))))))) - - - -(define parse-single-input (make-rule-parser single_input)) -(check-equal? - (syntax->datum - (parse-single-input - (adapt-python-tokenizer (open-input-string "def f(x):\n return x*x\n\n") - #:end-marker-to-eof? #t))) - '(single_input - (compound_stmt - (funcdef "def" "f" (parameters "(" (varargslist (fpdef "x")) ")") ":" (suite "\n" " " (stmt (simple_stmt (small_stmt (flow_stmt (return_stmt "return" (testlist (test (or_test (and_test (not_test (comparison (expr (xor_expr (and_expr (shift_expr (arith_expr (term (factor (power (atom "x"))) "*" (factor (power (atom "x")))))))))))))))))) "\n")) ""))) "\n")) diff --git a/brag/info.rkt b/brag/info.rkt index 45a38e2..6a6ac1a 100755 --- a/brag/info.rkt +++ b/brag/info.rkt @@ -1,6 +1,6 @@ #lang setup/infotab -(define deps '("base" "parser-tools-lib" "rackunit-lib" "python-tokenizer")) +(define deps '("base" "parser-tools-lib" "rackunit-lib")) (define build-deps '("at-exp-lib" "parser-tools-doc" "racket-doc" "scribble-lib")) (define collection 'multi)