oxaliq/feature-change-applier
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

define tokens for lexing set [] in, yield [] operations

Browse source
This commit is contained in:
Sorrel Bri 2020-03-14 23:06:15 -07:00
parent 2634e35a01
commit a7dad0d3e5
4 changed files with 97 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
public/latl/ipa.latl
View file

@ -71,7 +71,7 @@ set VOWELS = { ORAL_VOWELS or NASAL_VOWELS }
set PHONES = { VOWELS or CONSONANTS } set PHONES = { VOWELS or CONSONANTS }
print [ GLOBAL ] ; print [ GLOBAL ]
[lateral [lateral
+= +=

4
src/utils/latl/lexer.js
View file

@ -98,6 +98,10 @@ export const lexer = moo.states({
}, },
identifier: /[A-Z]+[A-Z_]+/, identifier: /[A-Z]+[A-Z_]+/,
whiteSpace: /[\t ]+/, whiteSpace: /[\t ]+/,
openSquareBracket: /\]/,
closeSquareBracket: /\[/,
identifier: /[A-Z]+[A-Z_]*/,
phone: /[\u00c0-\u03FFa-z]+/,
newLine: { match: /\n/, lineBreaks: true } newLine: { match: /\n/, lineBreaks: true }
}, },

86
src/utils/latl/test/assertionData.js
View file

@ -208,4 +208,90 @@ set CLICK_CONSONANTS = { TENUIS_CLICK_CONSONANTS or VOICED_CLICK_CONSONANTS
{ type: 'closeCurlyBracket', value: '}'} { type: 'closeCurlyBracket', value: '}'}
] ]
}, },
setDefinitionYield: {
latl: `
set NASAL_VOWELS = { [ V ] in ORAL_VOWELS yield [ Ṽ ] },
SHORT_NASAL_VOWELS = { [ Vː ] in NASAL_VOWELS yield [ V ]ː },
LONG_NASAL_VOWELS = { [ Vː ] in NASAL_VOWELS }`,
tokens: [
{ type: 'whiteSpace', value: '\n', },
{ type: 'kw-set', value: 'set', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'setIdentifier', value: 'NASAL_VOWELS', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'equal', value: '=', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'openCurlyBracket', value: '{', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'closeSquareBracket', value: '[', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'identifier', value: 'V', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'openSquareBracket', value: ']', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'kw-set-in', value: 'in', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'identifier', value: 'ORAL_VOWELS', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'kw-set-yield', value: 'yield', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'closeSquareBracket', value: '[', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'identifier', value: 'V', },
{ type: 'phone', value: '̃', }, // test display for COMBINING TILDE OVERLAY is deceiving
{ type: 'whiteSpace', value: ' ', },
{ type: 'openSquareBracket', value: ']', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'closeCurlyBracket', value: '}', },
{ type: 'comma', value: ',', },
{ type: 'whiteSpace', value: '\n ', },
{ type: 'setIdentifier', value: 'SHORT_NASAL_VOWELS', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'equal', value: '=', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'openCurlyBracket', value: '{', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'closeSquareBracket', value: '[', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'identifier', value: 'V', },
{ type: 'phone', value: 'ː', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'openSquareBracket', value: ']', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'kw-set-in', value: 'in', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'identifier', value: 'NASAL_VOWELS', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'kw-set-yield', value: 'yield', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'closeSquareBracket', value: '[', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'identifier', value: 'V', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'openSquareBracket', value: ']', },
{ type: 'phone', value: 'ː', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'closeCurlyBracket', value: '}', },
{ type: 'comma', value: ',', },
{ type: 'whiteSpace', value: '\n ', },
{ type: 'setIdentifier', value: 'LONG_NASAL_VOWELS', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'equal', value: '=', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'openCurlyBracket', value: '{', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'closeSquareBracket', value: '[', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'identifier', value: 'V', },
{ type: 'phone', value: 'ː', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'openSquareBracket', value: ']', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'kw-set-in', value: 'in', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'identifier', value: 'NASAL_VOWELS', },
{ type: 'whiteSpace', value: ' ', },
{ type: 'closeCurlyBracket', value: '}', },
]
}
} }

6
src/utils/latl/test/lexer.test.js
View file

@ -49,5 +49,11 @@ describe('lexer', () => {
const { latl, tokens } = assertionData.setDefinitionJoin; const { latl, tokens } = assertionData.setDefinitionJoin;
const stream = getStream(latl); const stream = getStream(latl);
expect(stream).toStrictEqual(tokens); expect(stream).toStrictEqual(tokens);
});
it('lexes set definition with yield operation', () => {
const { latl, tokens } = assertionData.setDefinitionYield;
const stream = getStream(latl);
expect(stream).toStrictEqual(tokens);
}) })
}) })