From 598c65e45a3f9ad09f13359d381a15a00b375cb5 Mon Sep 17 00:00:00 2001 From: Matthew Butterick Date: Thu, 29 Jun 2017 08:51:53 -0700 Subject: [PATCH] versioned struct tests passing --- pitfall/restructure/generic.rkt | 6 +- pitfall/restructure/struct-test.rkt | 10 +- pitfall/restructure/versioned-struct-test.rkt | 215 +++++++++++++++++- pitfall/restructure/versioned-struct.rkt | 51 +++-- 4 files changed, 246 insertions(+), 36 deletions(-) diff --git a/pitfall/restructure/generic.rkt b/pitfall/restructure/generic.rkt index b67a19f8..a0cef86d 100644 --- a/pitfall/restructure/generic.rkt +++ b/pitfall/restructure/generic.rkt @@ -13,13 +13,13 @@ (ref-set! indexable i v) (ref-keys indexable) #:defaults - ([hash? (define ref hash-ref) + ([hash? (define (ref o i) (hash-ref o i #f)) (define ref-set! hash-set!) (define ref-keys hash-keys)] - [dict? (define ref dict-ref) + [dict? (define (ref o i) (dict-ref o i #f)) (define ref-set! dict-set!) (define ref-keys dict-keys)] - [object? (define (ref o i) (with-handlers ([exn:fail:object? (λ (exn) (hash-ref (get-field _hash o) i))]) (dynamic-get-field i o))) + [object? (define (ref o i) (with-handlers ([exn:fail:object? (λ (exn) (hash-ref (get-field _hash o) i #f))]) (dynamic-get-field i o))) (define (ref-set! o i v) (with-handlers ([exn:fail:object? (λ (exn) (hash-set! (get-field _hash o) i v))]) (dynamic-set-field! i o v))) (define (ref-keys o) (append (remove '_hash (field-names o)) (hash-keys (get-field _hash o))))])) diff --git a/pitfall/restructure/struct-test.rkt b/pitfall/restructure/struct-test.rkt index d204dfe4..f54b1b42 100644 --- a/pitfall/restructure/struct-test.rkt +++ b/pitfall/restructure/struct-test.rkt @@ -19,7 +19,7 @@ https://github.com/mbutterick/restructure/blob/master/test/Struct.coffee ; name: 'devon' ; age: 21 -(let ([stream (+DecodeStream (+Buffer (bytes->list (bytes-append (bytes #x05) #"devon" (bytes #x15)))))] +(let ([stream (+DecodeStream (+Buffer "\x05devon\x15"))] [struct (+Struct (dictify 'name (+StringT uint8) 'age uint8))]) (check-equal? (send (send struct decode stream) ht) @@ -41,7 +41,7 @@ https://github.com/mbutterick/restructure/blob/master/test/Struct.coffee ; age: 32 ; canDrink: true -(let ([stream (+DecodeStream (+Buffer (bytes->list (bytes-append (bytes #x05) #"devon" (bytes #x20)))))] +(let ([stream (+DecodeStream (+Buffer "\x05devon\x20"))] [struct (+Struct (dictify 'name (+StringT uint8) 'age uint8))]) (set-field! process struct (λ (o stream) (ref-set! o 'canDrink (>= (ref o 'age) 21)))) @@ -62,7 +62,7 @@ https://github.com/mbutterick/restructure/blob/master/test/Struct.coffee ; age: 32 ; canDrink: true -(let ([stream (+DecodeStream (+Buffer (bytes->list (bytes-append (bytes #x05) #"devon" (bytes #x20)))))] +(let ([stream (+DecodeStream (+Buffer "\x05devon\x20"))] [struct (+Struct (dictify 'name (+StringT uint8) 'age uint8 'canDrink (λ (o) (>= (ref o 'age) 21))))]) @@ -148,7 +148,7 @@ https://github.com/mbutterick/restructure/blob/master/test/Struct.coffee ; ; stream.end() -(let ([stream (+DecodeStream (+Buffer (bytes->list (bytes-append (bytes #x05) #"devon" (bytes #x15)))))] +(let ([stream (+DecodeStream (+Buffer "\x05devon\x15"))] [struct (+Struct (dictify 'name (+StringT uint8) 'age uint8))]) (check-equal? (send (send struct decode stream) ht) @@ -182,7 +182,7 @@ https://github.com/mbutterick/restructure/blob/master/test/Struct.coffee (set-field! preEncode struct (λ (val stream) (ref-set! val 'nameLength (length (ref val 'name))))) (send struct encode stream (mhasheq 'name "devon" 'age 21)) (check-equal? (send stream dump) - (+Buffer (bytes->list (bytes-append (bytes #x05) #"devon" (bytes #x15)))))) + (+Buffer "\x05devon\x15"))) ; todo: when pointer is ready diff --git a/pitfall/restructure/versioned-struct-test.rkt b/pitfall/restructure/versioned-struct-test.rkt index 093dc1a4..75700fac 100644 --- a/pitfall/restructure/versioned-struct-test.rkt +++ b/pitfall/restructure/versioned-struct-test.rkt @@ -39,12 +39,12 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe 'age uint8 'gender uint8)))]) - (let ([stream (+DecodeStream (+Buffer "\u0000\u0005devon\u0015"))]) + (let ([stream (+DecodeStream (+Buffer "\x00\x05devon\x15"))]) (check-equal? (send (send struct decode stream) ht) (mhasheq 'name "devon" 'age 21 'version 0))) - (let ([stream (+DecodeStream (+Buffer "\u0001\u000adevon 👍\u0015\u0000"))]) + (let ([stream (+DecodeStream (+Buffer "\x01\x0adevon 👍\x15\x00"))]) (check-equal? (send (send struct decode stream) ht) (mhasheq 'name "devon 👍" 'age 21 'version 1 @@ -76,7 +76,7 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe 'age uint8 'gender uint8)))]) - (let ([stream (+DecodeStream (+Buffer "\u0005\u0005devon\u0015"))]) + (let ([stream (+DecodeStream (+Buffer "\x05\x05devon\x15"))]) (check-exn exn:fail:contract? (λ () (send struct decode stream))))) ; @@ -115,13 +115,13 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe 1 (dictify 'name (+StringT uint8 'utf8) 'gender uint8)))]) - (let ([stream (+DecodeStream (+Buffer "\u0000\u0015\u0001\u0005devon"))]) + (let ([stream (+DecodeStream (+Buffer "\x00\x15\x01\x05devon"))]) (check-equal? (send (send struct decode stream) ht) (mhasheq 'name "devon" 'age 21 'alive 1 'version 0))) - (let ([stream (+DecodeStream (+Buffer "\u0001\u0015\u0001\u000adevon 👍\u0000"))]) + (let ([stream (+DecodeStream (+Buffer "\x01\x15\x01\x0adevon 👍\x00"))]) (check-equal? (send (send struct decode stream) ht) (mhasheq 'name "devon 👍" 'age 21 'version 1 @@ -160,12 +160,12 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe 'age uint8 'gender uint8)))]) - (let ([stream (+DecodeStream (+Buffer "\u0005devon\u0015"))]) + (let ([stream (+DecodeStream (+Buffer "\x05devon\x15"))]) (check-equal? (send (send struct decode stream (mhash 'version 0)) ht) (mhasheq 'name "devon" 'age 21 'version 0))) - (let ([stream (+DecodeStream (+Buffer "\u000adevon 👍\u0015\u0000" 'utf8))]) + (let ([stream (+DecodeStream (+Buffer "\x0adevon 👍\x15\x00" 'utf8))]) (check-equal? (send (send struct decode stream (mhash 'version 1)) ht) (mhasheq 'name "devon 👍" 'age 21 'version 1 @@ -200,6 +200,33 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; version: 1 ; name: 'ice cream' ; isDesert: 1 + + +(let ([struct (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8 'ascii) + 'age uint8) + 1 (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8)) + 1 (dictify 'name (+StringT uint8) + 'isDessert uint8)))))]) + + (let ([stream (+DecodeStream (+Buffer "\x00\x05devon\x15"))]) + (check-equal? (send (send struct decode stream (mhash 'version 0)) ht) (mhasheq 'name "devon" + 'age 21 + 'version 0))) + + (let ([stream (+DecodeStream (+Buffer "\x01\x00\x05pasta"))]) + (check-equal? (send (send struct decode stream (mhash 'version 0)) ht) (mhasheq 'name "pasta" + 'version 0))) + + (let ([stream (+DecodeStream (+Buffer "\x01\x01\x09ice cream\x01"))]) + (check-equal? (send (send struct decode stream (mhash 'version 0)) ht) (mhasheq 'name "ice cream" + 'isDessert 1 + 'version 1)))) + + ; ; it 'should support process hook', -> ; struct = new VersionedStruct uint8, @@ -220,6 +247,22 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; name: 'devon' ; age: 21 ; processed: true + + +(let ([struct (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8 'ascii) + 'age uint8) + 1 (dictify 'name (+StringT uint8 'utf8) + 'age uint8 + 'gender uint8)))]) + (set-field! process struct (λ (o stream) (ref-set! o 'processed "true"))) + (let ([stream (+DecodeStream (+Buffer "\x00\x05devon\x15"))]) + (check-equal? (send (send struct decode stream) ht) (mhasheq 'name "devon" + 'processed "true" + 'age 21 + 'version 0)))) + ; ; describe 'size', -> ; it 'should compute the correct size', -> @@ -246,6 +289,25 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; gender: 0 ; ; size.should.equal 14 + +(let ([struct (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8 'ascii) + 'age uint8) + 1 (dictify 'name (+StringT uint8 'utf8) + 'age uint8 + 'gender uint8)))]) + + (check-equal? (send struct size (mhasheq 'name "devon" + 'age 21 + 'version 0)) 8) + + (check-equal? (send struct size (mhasheq 'name "devon 👍" + 'gender 0 + 'age 21 + 'version 1)) 14)) + + ; ; it 'should throw for unknown version', -> ; struct = new VersionedStruct uint8, @@ -262,6 +324,20 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; version: 5 ; name: 'devon' ; age: 21 + +(let ([struct (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8 'ascii) + 'age uint8) + 1 (dictify 'name (+StringT uint8 'utf8) + 'age uint8 + 'gender uint8)))]) + + (check-exn exn:fail:contract? (λ () (send struct size (mhasheq 'name "devon" + 'age 21 + 'version 5))))) + + ; ; it 'should support common header block', -> ; struct = new VersionedStruct uint8, @@ -290,7 +366,28 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; gender: 0 ; ; size.should.equal 15 -; + +(let ([struct (+VersionedStruct uint8 + (dictify + 'header (dictify 'age uint8 + 'alive uint8) + 0 (dictify 'name (+StringT uint8 'ascii)) + 1 (dictify 'name (+StringT uint8 'utf8) + 'gender uint8)))]) + + (check-equal? (send struct size (mhasheq 'name "devon" + 'age 21 + 'alive 1 + 'version 0)) 9) + + (check-equal? (send struct size (mhasheq 'name "devon 👍" + 'gender 0 + 'age 21 + 'alive 1 + 'version 1)) 15)) + + +; todo: pointers ; it 'should compute the correct size with pointers', -> ; struct = new VersionedStruct uint8, ; 0: @@ -308,6 +405,10 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; ptr: 'hello' ; ; size.should.equal 15 + +(displayln "warning: pointer test not done in versioned-struct-test") + + ; ; it 'should throw if no value is given', -> ; struct = new VersionedStruct uint8, @@ -322,6 +423,18 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; should.throw -> ; struct.size() ; , /not a fixed size/i + + +(let ([struct (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8 'ascii) + 'age uint8) + 1 (dictify 'name (+StringT uint8 'utf8) + 'age uint8 + 'gender uint8)))]) + + (check-exn exn:fail:contract? (λ () (send struct size)))) + ; ; describe 'encode', -> ; it 'should encode objects to buffers', (done) -> @@ -351,6 +464,25 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; gender: 0 ; ; stream.end() + +(let ([struct (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8 'ascii) + 'age uint8) + 1 (dictify 'name (+StringT uint8 'utf8) + 'age uint8 + 'gender uint8)))] + [stream (+EncodeStream)]) + (send struct encode stream (mhasheq 'name "devon" + 'age 21 + 'version 0)) + (send struct encode stream (mhasheq 'name "devon 👍" + 'age 21 + 'gender 0 + 'version 1)) + (check-equal? (send stream dump) (+Buffer "\x00\x05devon\x15\x01\x0adevon 👍\x15\x00" 'utf8))) + + ; ; it 'should throw for unknown version', -> ; struct = new VersionedStruct uint8, @@ -368,6 +500,19 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; version: 5 ; name: 'devon' ; age: 21 + +(let ([struct (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8 'ascii) + 'age uint8) + 1 (dictify 'name (+StringT uint8 'utf8) + 'age uint8 + 'gender uint8)))] + [stream (+EncodeStream)]) + (check-exn exn:fail:contract? (λ () (send struct encode stream (mhasheq 'name "devon" + 'age 21 + 'version 5))))) + ; ; it 'should support common header block', (done) -> ; struct = new VersionedStruct uint8, @@ -399,7 +544,32 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; gender: 0 ; ; stream.end() -; + + +(let ([struct (+VersionedStruct uint8 + (dictify + 'header (dictify 'age uint8 + 'alive uint8) + 0 (dictify 'name (+StringT uint8 'ascii)) + 1 (dictify 'name (+StringT uint8 'utf8) + 'gender uint8)))] + [stream (+EncodeStream)]) + + (send struct encode stream (mhasheq 'name "devon" + 'age 21 + 'alive 1 + 'version 0)) + + (send struct encode stream (mhasheq 'name "devon 👍" + 'gender 0 + 'age 21 + 'alive 1 + 'version 1)) + + (check-equal? (send stream dump) (+Buffer "\x00\x15\x01\x05devon\x01\x15\x01\x0adevon 👍\x00" 'utf8))) + + +; todo ; it 'should encode pointer data after structure', (done) -> ; struct = new VersionedStruct uint8, ; 0: @@ -409,6 +579,7 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; name: new StringT uint8, 'utf8' ; age: uint8 ; ptr: new Pointer uint8, new StringT uint8 + ; ; stream = new EncodeStream ; stream.pipe concat (buf) -> @@ -422,6 +593,12 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; ptr: 'hello' ; ; stream.end() + + +(displayln "warning: pointer test not done in versioned-struct-test") + + + ; ; it 'should support preEncode hook', (done) -> ; struct = new VersionedStruct uint8, @@ -450,4 +627,22 @@ https://github.com/mbutterick/restructure/blob/master/test/VersionedStruct.coffe ; age: 21 ; gender: 0 ; -; stream.end() \ No newline at end of file +; stream.end() + + +(let ([struct (+VersionedStruct uint8 + (dictify + 0 (dictify 'name (+StringT uint8 'ascii) + 'age uint8) + 1 (dictify 'name (+StringT uint8 'utf8) + 'age uint8 + 'gender uint8)))] + [stream (+EncodeStream)]) + (set-field! preEncode struct (λ (val stream) (ref-set! val 'version (if (ref val 'gender) 1 0)))) + (send struct encode stream (mhasheq 'name "devon" + 'age 21 + 'version 0)) + (send struct encode stream (mhasheq 'name "devon 👍" + 'age 21 + 'gender 0)) + (check-equal? (send stream dump) (+Buffer "\x00\x05devon\x15\x01\x0adevon 👍\x15\x00" 'utf8))) \ No newline at end of file diff --git a/pitfall/restructure/versioned-struct.rkt b/pitfall/restructure/versioned-struct.rkt index fd16f878..85781e3c 100644 --- a/pitfall/restructure/versioned-struct.rkt +++ b/pitfall/restructure/versioned-struct.rkt @@ -35,10 +35,10 @@ https://github.com/mbutterick/restructure/blob/master/src/VersionedStruct.coffee (versionGetter parent)] [else (send type decode stream)])) - (when (dict-ref versions 'header #f) + (when (ref versions 'header) (_parseFields stream res (ref versions 'header))) - (define fields (dict-ref versions (ref res 'version) (λ () (raise-argument-error 'VersionedStruct:decode "valid version key" (cons version (· this versions)))))) + (define fields (or (ref versions (ref res 'version)) (raise-argument-error 'VersionedStruct:decode "valid version key" (cons version (· this versions))))) (cond [(VersionedStruct? fields) (send fields decode stream parent)] @@ -50,22 +50,37 @@ https://github.com/mbutterick/restructure/blob/master/src/VersionedStruct.coffee (define/public-final (force-version! version) (set! forced-version version)) - (define/override (encode stream input-hash [parent #f]) - (unless (hash? input-hash) - (raise-argument-error 'Struct:encode "hash" input-hash)) + (define/override (encode stream val [parent #f]) + (unless (hash? val) + (raise-argument-error 'Struct:encode "hash" val)) - (send this preEncode input-hash stream) ; preEncode goes first, because it might bring input hash into compliance + (send this preEncode val stream) ; preEncode goes first, because it might bring input hash into compliance - (define fields (dict-ref versions (· input-hash version) (λ () (raise-argument-error 'VersionedStruct:encode "valid version key" version)))) + (define ctx (mhash 'pointers empty + 'startOffset (· stream pos) + 'parent parent + 'val val + 'pointerSize 0)) - (unless (andmap (λ (key) (member key (hash-keys input-hash))) (dict-keys fields)) - (raise-argument-error 'Struct:encode (format "hash that contains superset of Struct keys: ~a" (dict-keys fields)) (hash-keys input-hash))) + (ref-set! ctx 'pointerOffset (+ (· stream pos) (size val ctx #f))) - (cond - [(dict? fields) - (for* ([(key type) (in-dict fields)]) - (send type encode stream (hash-ref input-hash key)))] - [else (send fields encode stream input-hash parent)])) + (when (not (symbol? type)) + (send type encode stream (· val version))) + + (when (ref versions 'header) + (for ([(key type) (in-dict (ref versions 'header))]) + (send type encode stream (ref val key) ctx))) + + (define fields (or (ref versions (· val version)) (raise-argument-error 'VersionedStruct:encode "valid version key" version))) + + (unless (andmap (λ (key) (member key (ref-keys val))) (ref-keys fields)) + (raise-argument-error 'VersionedStruct:encode (format "hash that contains superset of Struct keys: ~a" (dict-keys fields)) (hash-keys val))) + + (for ([(key type) (in-dict fields)]) + (send type encode stream (ref val key) ctx)) + + (for ([ptr (in-list (ref ctx 'pointers))]) + (send (ref ptr 'type) encode stream (ref ptr 'val) (ref ptr 'parent)))) (define/override (size [val (mhash)] [parent #f] [includePointers #t]) @@ -77,7 +92,7 @@ https://github.com/mbutterick/restructure/blob/master/src/VersionedStruct.coffee 'pointerSize 0)) (define size 0) - (when (not (string? type)) + (when (not (symbol? type)) (increment! size (send type size (ref val 'version) ctx))) (when (ref versions 'header) @@ -85,10 +100,10 @@ https://github.com/mbutterick/restructure/blob/master/src/VersionedStruct.coffee (for/sum ([(key type) (in-dict (ref versions 'header))]) (send type size (ref val key) ctx)))) - (define fields (dict-ref versions (ref val 'version) (λ () (raise-argument-error 'VersionedStruct:encode "valid version key" version)))) + (define fields (or (ref versions (ref val 'version)) (raise-argument-error 'VersionedStruct:encode "valid version key" version))) (increment! size - (for/sum ([(key type) (in-dict (ref versions 'header))]) + (for/sum ([(key type) (in-dict fields)]) (send type size (ref val key) ctx))) (when includePointers @@ -111,7 +126,7 @@ https://github.com/mbutterick/restructure/blob/master/src/VersionedStruct.coffee (cons v (for/list ([num-type (in-list field-types)]) (cons (gensym) num-type))))) (define vs (+VersionedStruct which-struct struct-versions)) - (define struct-size (for/sum ([num-type (in-list (map cdr (dict-ref struct-versions which-struct)))]) + (define struct-size (for/sum ([num-type (in-list (map cdr (ref struct-versions which-struct)))]) (send num-type size))) (define bs (apply bytes (for/list ([i (in-range struct-size)]) (random 256))))