hack set definition postprocessors

This commit is contained in:
Sorrel Bri 2020-05-07 23:24:19 -07:00
parent dee27b0d30
commit 3d4d1cd66e
5 changed files with 172 additions and 40 deletions

View file

@ -6,7 +6,7 @@ function id(x) { return x[0]; }
const { lexer } = require('./lexer.js'); const { lexer } = require('./lexer.js');
const getTerminal = d => d ? d[0] : null; const getTerminal = d => d ? d[0] : null;
const getAll = d => d.map((item, i) => ({ [i]: item })); const getAll = d => d.map((item, i) => ({ [i]: item }));
const flag = token => d => d.map(item => ({ [token]: clearNull(item) })) const flag = token => d => d.map(item => ({ [token]: item }))
const clearNull = d => d.filter(t => !!t && (t.length !== 1 || t[0])).map(t => t.length ? clearNull(t) : t); const clearNull = d => d.filter(t => !!t && (t.length !== 1 || t[0])).map(t => t.length ? clearNull(t) : t);
const flagIndex = d => d.map((item, i) => ({[i]: item})) const flagIndex = d => d.map((item, i) => ({[i]: item}))
const remove = _ => null; const remove = _ => null;
@ -35,22 +35,21 @@ var grammar = {
{"name": "_", "symbols": ["_$ebnf$1"], "postprocess": remove}, {"name": "_", "symbols": ["_$ebnf$1"], "postprocess": remove},
{"name": "__", "symbols": [(lexer.has("whiteSpace") ? {type: "whiteSpace"} : whiteSpace)], "postprocess": remove}, {"name": "__", "symbols": [(lexer.has("whiteSpace") ? {type: "whiteSpace"} : whiteSpace)], "postprocess": remove},
{"name": "statement", "symbols": ["comment"]}, {"name": "statement", "symbols": ["comment"]},
{"name": "statement", "symbols": ["definition"], "postprocess": pipe(getTerminal, clearNull)}, {"name": "statement", "symbols": ["definition"], "postprocess": pipe(getTerminal)},
{"name": "comment", "symbols": [(lexer.has("comment") ? {type: "comment"} : comment)], "postprocess": pipe(getTerminal, remove)}, {"name": "comment", "symbols": [(lexer.has("comment") ? {type: "comment"} : comment)], "postprocess": pipe(getTerminal, remove)},
{"name": "definition", "symbols": [(lexer.has("kwSet") ? {type: "kwSet"} : kwSet), "__", "setDefinition"], "postprocess": d => ({token: d[0].type, [d[0].value]: d[2]})}, {"name": "definition", "symbols": [(lexer.has("kwSet") ? {type: "kwSet"} : kwSet), "__", "setDefinition"], "postprocess": d => ({[d[0].value]: d[2]})},
{"name": "setDefinition$ebnf$1", "symbols": []}, {"name": "setDefinition$ebnf$1", "symbols": []},
{"name": "setDefinition$ebnf$1$subexpression$1", "symbols": [(lexer.has("setIdentifier") ? {type: "setIdentifier"} : setIdentifier), "__", (lexer.has("equal") ? {type: "equal"} : equal), "__", "setExpression", (lexer.has("comma") ? {type: "comma"} : comma), "__"]}, {"name": "setDefinition$ebnf$1$subexpression$1", "symbols": [(lexer.has("setIdentifier") ? {type: "setIdentifier"} : setIdentifier), "__", (lexer.has("equal") ? {type: "equal"} : equal), "__", "setExpression", (lexer.has("comma") ? {type: "comma"} : comma), "__"]},
{"name": "setDefinition$ebnf$1", "symbols": ["setDefinition$ebnf$1", "setDefinition$ebnf$1$subexpression$1"], "postprocess": function arrpush(d) {return d[0].concat([d[1]]);}}, {"name": "setDefinition$ebnf$1", "symbols": ["setDefinition$ebnf$1", "setDefinition$ebnf$1$subexpression$1"], "postprocess": function arrpush(d) {return d[0].concat([d[1]]);}},
{"name": "setDefinition", "symbols": ["setDefinition$ebnf$1", (lexer.has("setIdentifier") ? {type: "setIdentifier"} : setIdentifier), "__", (lexer.has("equal") ? {type: "equal"} : equal), "__", "setExpression"]}, {"name": "setDefinition", "symbols": ["setDefinition$ebnf$1", (lexer.has("setIdentifier") ? {type: "setIdentifier"} : setIdentifier), "__", (lexer.has("equal") ? {type: "equal"} : equal), "__", "setExpression"], "postprocess": d => {
{"name": "setExpression", "symbols": [(lexer.has("openSquareBracket") ? {type: "openSquareBracket"} : openSquareBracket), "_", "phoneList", "_", (lexer.has("closeSquareBracket") ? {type: "closeSquareBracket"} : closeSquareBracket)], "postprocess": d => d.filter(t => t && t.length)}, if (d.type === 'setIdentifier') return { setIdentifier: d.value }
return d
} },
{"name": "setExpression", "symbols": [(lexer.has("openSquareBracket") ? {type: "openSquareBracket"} : openSquareBracket), "_", "phoneList", "_", (lexer.has("closeSquareBracket") ? {type: "closeSquareBracket"} : closeSquareBracket)]},
{"name": "phoneList$ebnf$1", "symbols": []}, {"name": "phoneList$ebnf$1", "symbols": []},
{"name": "phoneList$ebnf$1$subexpression$1", "symbols": [(lexer.has("phone") ? {type: "phone"} : phone), (lexer.has("comma") ? {type: "comma"} : comma), "_"]}, {"name": "phoneList$ebnf$1$subexpression$1", "symbols": [(lexer.has("phone") ? {type: "phone"} : phone), (lexer.has("comma") ? {type: "comma"} : comma), "_"]},
{"name": "phoneList$ebnf$1", "symbols": ["phoneList$ebnf$1", "phoneList$ebnf$1$subexpression$1"], "postprocess": function arrpush(d) {return d[0].concat([d[1]]);}}, {"name": "phoneList$ebnf$1", "symbols": ["phoneList$ebnf$1", "phoneList$ebnf$1$subexpression$1"], "postprocess": function arrpush(d) {return d[0].concat([d[1]]);}},
{"name": "phoneList", "symbols": ["phoneList$ebnf$1", (lexer.has("phone") ? {type: "phone"} : phone)], "postprocess": d => d.filter(t => t && (t.type === 'phone' || t[0]) ) {"name": "phoneList", "symbols": ["phoneList$ebnf$1", (lexer.has("phone") ? {type: "phone"} : phone)], "postprocess": pipe(d => d ? d.toString() : d)}
.flatMap(t => {
if (!t.length) return t;
return t[0].filter(st => st && st.type === 'phone')
}) }
] ]
, ParserStart: "main" , ParserStart: "main"
} }

View file

@ -2,7 +2,7 @@
const { lexer } = require('./lexer.js'); const { lexer } = require('./lexer.js');
const getTerminal = d => d ? d[0] : null; const getTerminal = d => d ? d[0] : null;
const getAll = d => d.map((item, i) => ({ [i]: item })); const getAll = d => d.map((item, i) => ({ [i]: item }));
const flag = token => d => d.map(item => ({ [token]: clearNull(item) })) const flag = token => d => d.map(item => ({ [token]: item }))
const clearNull = d => d.filter(t => !!t && (t.length !== 1 || t[0])).map(t => t.length ? clearNull(t) : t); const clearNull = d => d.filter(t => !!t && (t.length !== 1 || t[0])).map(t => t.length ? clearNull(t) : t);
const flagIndex = d => d.map((item, i) => ({[i]: item})) const flagIndex = d => d.map((item, i) => ({[i]: item}))
const remove = _ => null; const remove = _ => null;
@ -32,24 +32,25 @@ __ -> %whiteSpace
{% remove %} {% remove %}
statement -> comment | definition statement -> comment | definition
{% pipe(getTerminal, clearNull) %} {% pipe(getTerminal) %}
comment -> %comment comment -> %comment
{% pipe(getTerminal, remove) %} {% pipe(getTerminal, remove) %}
# SETS # SETS
definition -> %kwSet __ setDefinition definition -> %kwSet __ setDefinition
{% d => ({token: d[0].type, [d[0].value]: d[2]}) %} {% d => ({[d[0].value]: d[2]}) %}
setDefinition -> (%setIdentifier __ %equal __ setExpression %comma __):* %setIdentifier __ %equal __ setExpression setDefinition -> (%setIdentifier __ %equal __ setExpression %comma __):* %setIdentifier __ %equal __ setExpression
# {% pipe( {% d => {
# //constructSet, if (d.type === 'setIdentifier') return { setIdentifier: d.value }
# getTerminal) %} return d
} %}
setExpression -> %openSquareBracket _ phoneList _ %closeSquareBracket setExpression -> %openSquareBracket _ phoneList _ %closeSquareBracket
{% d => d.filter(t => t && t.length) %} # {% pipe(d => d.filter(t => t && t.length)) %}
phoneList -> (%phone %comma _):* %phone phoneList -> (%phone %comma _):* %phone
# {% clearNull %} {% pipe(d => d ? d.toString() : d) %}
{% d => d.filter(t => t && (t.type === 'phone' || t[0]) ) # {% d => d.filter(t => t && (t.type === 'phone' || t[0]) )
.flatMap(t => { # .flatMap(t => {
if (!t.length) return t; # if (!t.length) return t;
return t[0].filter(st => st && st.type === 'phone') # return t[0].filter(st => st && st.type === 'phone')
}) %} # }) %}

View file

@ -4,6 +4,9 @@ export const assertionData = {
tokens: [ tokens: [
{ type: 'comment', value: '; comment'} { type: 'comment', value: '; comment'}
], ],
AST: {
main: []
},
code: '' code: ''
}, },
simpleSetDefinition: { simpleSetDefinition: {
@ -12,7 +15,8 @@ export const assertionData = {
{ type: 'kwSet', value: 'set' }, { type: 'kwSet', value: 'set' },
{ type: 'whiteSpace', value: ' ' }, { type: 'whiteSpace', value: ' ' },
{ type: 'setIdentifier', value: 'PLOSIVES' } { type: 'setIdentifier', value: 'PLOSIVES' }
] ],
code: ''
}, },
commaSetDefinition: { commaSetDefinition: {
latl: ` latl: `
@ -129,7 +133,21 @@ set NASAL_PULMONIC_CONSONANTS = [ m̥, m, ɱ, n̼, n̥, n, ɳ̊,
{ type: 'phone', value: 'ʔ' }, { type: 'phone', value: 'ʔ' },
{ type: 'whiteSpace', value: ' ' }, { type: 'whiteSpace', value: ' ' },
{ type: 'closeSquareBracket', value: ']' } { type: 'closeSquareBracket', value: ']' }
] ],
AST: {
main: [
{
set: {
setIdentifier: 'NASAL_PULMONIC_CONSONANTS',
items: [ 'm̥', 'm', 'ɱ', 'n̼', 'n̥', 'n', 'ɳ̊', 'ɳ', 'ɲ̊', 'ɲ', `ŋ`, ' ̊ŋ', 'ɴ' ]
},
set: {
setIdentifier: 'STOP_PULMONIC_CONSONANTS',
items: [ 'p', 'b', 'p̪', 'b̪', 't̼', 'd̼', 't', 'd', 'ʈ', 'ɖ', 'c', 'ɟ', 'k', 'ɡ', 'q', 'ɢ', 'ʡ', 'ʔ' ]
}
}
]
}
}, },
setAliasDefinition: { setAliasDefinition: {
latl: ` latl: `

View file

@ -3,10 +3,8 @@ import { codeGenerator } from '../codeGenerator';
describe('codeGenerator', () => { describe('codeGenerator', () => {
it('parses simple comment', () => { it('parses simple comment', () => {
const { latl } = assertionData.simpleComment; const { latl, code } = assertionData.simpleComment;
const code = codeGenerator(latl) const generatedCode = codeGenerator(latl);
// expect(AST.length).toBe(1); expect(generatedCode).toEqual(code);
// expect(AST[0]).toStrictEqual({ main: [ ]}) });
console.log(code)
})
}) })

View file

@ -4,17 +4,18 @@ import { assertionData } from './assertionData';
describe('parser', () => { describe('parser', () => {
it('parses simple comment', () => { it('parses simple comment', () => {
const { latl } = assertionData.simpleComment; const { latl, AST } = assertionData.simpleComment;
const AST = parser().feed(latl).results; const feedResults = parser().feed(latl).results;
expect(AST.length).toBe(1); expect(feedResults.length).toBe(1);
expect(AST[0]).toStrictEqual({ main: [ ]}) expect(feedResults[0]).toStrictEqual(AST)
}) })
// it('parses multiple set definitions with comma operator', () => { it('parses multiple set definitions with comma operator', () => {
// const { latl } = assertionData.commaSetDefinition; const { latl, AST } = assertionData.commaSetDefinition;
// const AST = parser().feed(latl).results; const feedResults = parser().feed(latl).results;
// console.log(AST[0]) expect(feedResults.length).toBe(1);
// }); expect(feedResults[0]).toStrictEqual(AST);
});
// it('lexes set definition with alias', () => { // it('lexes set definition with alias', () => {
// const { latl, tokens } = assertionData.setAliasDefinition; // const { latl, tokens } = assertionData.setAliasDefinition;
@ -46,3 +47,118 @@ describe('parser', () => {
// expect(stream).toStrictEqual(tokens); // expect(stream).toStrictEqual(tokens);
// }) // })
}) })
// {
// "set":
// [
// [
// [
// {
// "col": 5,
// "line": 2,
// "lineBreaks": 0,
// "offset": 5,
// "text": "NASAL_PULMONIC_CONSONANTS",
// "toString": [tokenToString],
// "type": "setIdentifier",
// "value": "NASAL_PULMONIC_CONSONANTS",
// },
// null,
// {
// "col": 45,
// "line": 2,
// "lineBreaks": 0,
// "offset": 45,
// "text": "=",
// "toString": [tokenToString],
// "type": "equal",
// "value": "=",
// },
// null,
// [
// [
// {
// "col": 49,
// "line": 2,
// "lineBreaks": 0,
// "offset": 49,
// "text": "m̥",
// "toString": [tokenToString],
// "type": "phone",
// "value": "m̥",
// },
// {
// "col": 91,
// "line": 2,
// "lineBreaks": 0,
// "offset": 91,
// "text": "ɴ",
// "toString": [tokenToString],
// "type": "phone",
// "value": "ɴ",
// },
// ],
// ],
// {
// "col": 94,
// "line": 2,
// "lineBreaks": 0,
// "offset": 94,
// "text": ",",
// "toString": [tokenToString],
// "type": "comma",
// "value": ",",
// },
// null,
// ],
// ],
// - "setIdentifier": "STOP_PULMONIC_CONSONANTS",
// {
// "col": 5,
// "line": 3,
// "lineBreaks": 0,
// "offset": 100,
// "text": "STOP_PULMONIC_CONSONANTS",
// "toString": [tokenToString],
// "type": "setIdentifier",
// "value": "STOP_PULMONIC_CONSONANTS",
// },
// null,
// {
// "col": 45,
// "line": 3,
// "lineBreaks": 0,
// "offset": 140,
// "text": "=",
// "toString": [tokenToString],
// "type": "equal",
// "value": "=",
// },
// null,
// [
// [
// {
// "col": 49,
// "line": 3,
// "lineBreaks": 0,
// "offset": 144,
// "text": "p",
// "toString": [tokenToString],
// "type": "phone",
// "value": "p",
// },
// {
// "col": 104,
// "line": 3,
// "lineBreaks": 0,
// "offset": 199,
// "text": "ʔ",
// "toString": [tokenToString],
// "type": "phone",
// "value": "ʔ",
// },
// ],
// ],
// ],
// "token": "kwSet",
// }