From a7dad0d3e5cad2da2bb75f9116dc68459341b9cf Mon Sep 17 00:00:00 2001 From: Sorrel Bri Date: Sat, 14 Mar 2020 23:06:15 -0700 Subject: [PATCH] define tokens for lexing set [] in, yield [] operations --- public/latl/ipa.latl | 2 +- src/utils/latl/lexer.js | 4 ++ src/utils/latl/test/assertionData.js | 86 ++++++++++++++++++++++++++++ src/utils/latl/test/lexer.test.js | 6 ++ 4 files changed, 97 insertions(+), 1 deletion(-) diff --git a/public/latl/ipa.latl b/public/latl/ipa.latl index 19d35d5..3a02323 100644 --- a/public/latl/ipa.latl +++ b/public/latl/ipa.latl @@ -71,7 +71,7 @@ set VOWELS = { ORAL_VOWELS or NASAL_VOWELS } set PHONES = { VOWELS or CONSONANTS } -print [ GLOBAL ] +; print [ GLOBAL ] [lateral += diff --git a/src/utils/latl/lexer.js b/src/utils/latl/lexer.js index dd48a9c..d48d87b 100644 --- a/src/utils/latl/lexer.js +++ b/src/utils/latl/lexer.js @@ -98,6 +98,10 @@ export const lexer = moo.states({ }, identifier: /[A-Z]+[A-Z_]+/, whiteSpace: /[\t ]+/, + openSquareBracket: /\]/, + closeSquareBracket: /\[/, + identifier: /[A-Z]+[A-Z_]*/, + phone: /[\u00c0-\u03FFa-z]+/, newLine: { match: /\n/, lineBreaks: true } }, diff --git a/src/utils/latl/test/assertionData.js b/src/utils/latl/test/assertionData.js index 641e307..9e2badc 100644 --- a/src/utils/latl/test/assertionData.js +++ b/src/utils/latl/test/assertionData.js @@ -208,4 +208,90 @@ set CLICK_CONSONANTS = { TENUIS_CLICK_CONSONANTS or VOICED_CLICK_CONSONANTS { type: 'closeCurlyBracket', value: '}'} ] }, + setDefinitionYield: { + latl: ` +set NASAL_VOWELS = { [ V ] in ORAL_VOWELS yield [ Ṽ ] }, + SHORT_NASAL_VOWELS = { [ Vː ] in NASAL_VOWELS yield [ V ]ː }, + LONG_NASAL_VOWELS = { [ Vː ] in NASAL_VOWELS }`, + tokens: [ + { type: 'whiteSpace', value: '\n', }, + { type: 'kw-set', value: 'set', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'setIdentifier', value: 'NASAL_VOWELS', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'equal', value: '=', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'openCurlyBracket', value: '{', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'closeSquareBracket', value: '[', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'identifier', value: 'V', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'openSquareBracket', value: ']', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'kw-set-in', value: 'in', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'identifier', value: 'ORAL_VOWELS', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'kw-set-yield', value: 'yield', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'closeSquareBracket', value: '[', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'identifier', value: 'V', }, + { type: 'phone', value: '̃', }, // test display for COMBINING TILDE OVERLAY is deceiving + { type: 'whiteSpace', value: ' ', }, + { type: 'openSquareBracket', value: ']', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'closeCurlyBracket', value: '}', }, + { type: 'comma', value: ',', }, + { type: 'whiteSpace', value: '\n ', }, + { type: 'setIdentifier', value: 'SHORT_NASAL_VOWELS', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'equal', value: '=', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'openCurlyBracket', value: '{', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'closeSquareBracket', value: '[', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'identifier', value: 'V', }, + { type: 'phone', value: 'ː', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'openSquareBracket', value: ']', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'kw-set-in', value: 'in', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'identifier', value: 'NASAL_VOWELS', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'kw-set-yield', value: 'yield', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'closeSquareBracket', value: '[', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'identifier', value: 'V', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'openSquareBracket', value: ']', }, + { type: 'phone', value: 'ː', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'closeCurlyBracket', value: '}', }, + { type: 'comma', value: ',', }, + { type: 'whiteSpace', value: '\n ', }, + { type: 'setIdentifier', value: 'LONG_NASAL_VOWELS', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'equal', value: '=', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'openCurlyBracket', value: '{', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'closeSquareBracket', value: '[', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'identifier', value: 'V', }, + { type: 'phone', value: 'ː', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'openSquareBracket', value: ']', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'kw-set-in', value: 'in', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'identifier', value: 'NASAL_VOWELS', }, + { type: 'whiteSpace', value: ' ', }, + { type: 'closeCurlyBracket', value: '}', }, + ] + } } \ No newline at end of file diff --git a/src/utils/latl/test/lexer.test.js b/src/utils/latl/test/lexer.test.js index 0945da4..7f37d53 100644 --- a/src/utils/latl/test/lexer.test.js +++ b/src/utils/latl/test/lexer.test.js @@ -49,5 +49,11 @@ describe('lexer', () => { const { latl, tokens } = assertionData.setDefinitionJoin; const stream = getStream(latl); expect(stream).toStrictEqual(tokens); + }); + + it('lexes set definition with yield operation', () => { + const { latl, tokens } = assertionData.setDefinitionYield; + const stream = getStream(latl); + expect(stream).toStrictEqual(tokens); }) }) \ No newline at end of file