# # BSD 3-Clause License # # Copyright (c) 2018 - 2023, Oleg Malyavkin # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. extends Node const PROTO_VERSION_CONST : String = "const PROTO_VERSION = " const PROTO_VERSION_DEFAULT : String = PROTO_VERSION_CONST + "0" class Document: func _init(doc_name : String, doc_text : String): name = doc_name text = doc_text var name : String var text : String class TokenPosition: func _init(b : int, e : int): begin = b end = e var begin : int = 0 var end : int = 0 class Helper: class StringPosition: func _init(s : int, c : int, l : int): str_num = s column = c length = l var str_num : int var column : int var length : int static func str_pos(text : String, position : TokenPosition) -> StringPosition: var cur_str : int = 1 var cur_col : int = 1 var res_str : int = 0 var res_col : int = 0 var res_length : int = 0 for i in range(text.length()): if text[i] == "\n": cur_str += 1 cur_col = 0 if position.begin == i: res_str = cur_str res_col = cur_col res_length = position.end - position.begin + 1 break cur_col += 1 return StringPosition.new(res_str, res_col, res_length) static func text_pos(tokens : Array, index : int) -> TokenPosition: var res_begin : int = 0 var res_end : int = 0 if index < tokens.size() && index >= 0: res_begin = tokens[index].position.begin res_end = tokens[index].position.end return TokenPosition.new(res_begin, res_end) static func error_string(file_name, col, row, error_text): return file_name + ":" + str(col) + ":" + str(row) + ": error: " + error_text class AnalyzeResult: var classes : Array = [] var fields : Array = [] var groups : Array = [] var version : int = 0 var state : bool = false var tokens : Array = [] var syntax : Analysis.TranslationResult var imports : Array = [] var doc : Document func soft_copy() -> AnalyzeResult: var res : AnalyzeResult = AnalyzeResult.new() res.classes = classes res.fields = fields res.groups = groups res.version = version res.state = state res.tokens = tokens res.syntax = syntax res.imports = imports res.doc = doc return res class Analysis: func _init(path : String, doc : Document): path_dir = path document = doc var document : Document var path_dir : String const LEX = { LETTER = "[A-Za-z]", DIGIT_DEC = "[0-9]", DIGIT_OCT = "[0-7]", DIGIT_HEX = "[0-9]|[A-F]|[a-f]", BRACKET_ROUND_LEFT = "\\(", BRACKET_ROUND_RIGHT = "\\)", BRACKET_CURLY_LEFT = "\\{", BRACKET_CURLY_RIGHT = "\\}", BRACKET_SQUARE_LEFT = "\\[", BRACKET_SQUARE_RIGHT = "\\]", BRACKET_ANGLE_LEFT = "\\<", BRACKET_ANGLE_RIGHT = "\\>", SEMICOLON = ";", COMMA = ",", EQUAL = "=", SIGN = "\\+|\\-", SPACE = "\\s", QUOTE_SINGLE = "'", QUOTE_DOUBLE = "\"", } const TOKEN_IDENT : String = "(" + LEX.LETTER + "+" + "(" + LEX.LETTER + "|" + LEX.DIGIT_DEC + "|" + "_)*)" const TOKEN_FULL_IDENT : String = TOKEN_IDENT + "{0,1}(\\." + TOKEN_IDENT + ")+" const TOKEN_BRACKET_ROUND_LEFT : String = "(" + LEX.BRACKET_ROUND_LEFT + ")" const TOKEN_BRACKET_ROUND_RIGHT : String = "(" + LEX.BRACKET_ROUND_RIGHT + ")" const TOKEN_BRACKET_CURLY_LEFT : String = "(" + LEX.BRACKET_CURLY_LEFT + ")" const TOKEN_BRACKET_CURLY_RIGHT : String = "(" + LEX.BRACKET_CURLY_RIGHT + ")" const TOKEN_BRACKET_SQUARE_LEFT : String = "(" + LEX.BRACKET_SQUARE_LEFT + ")" const TOKEN_BRACKET_SQUARE_RIGHT : String = "(" + LEX.BRACKET_SQUARE_RIGHT + ")" const TOKEN_BRACKET_ANGLE_LEFT : String = "(" + LEX.BRACKET_ANGLE_LEFT + ")" const TOKEN_BRACKET_ANGLE_RIGHT : String = "(" + LEX.BRACKET_ANGLE_RIGHT + ")" const TOKEN_SEMICOLON : String = "(" + LEX.SEMICOLON + ")" const TOKEN_EUQAL : String = "(" + LEX.EQUAL + ")" const TOKEN_SIGN : String = "(" + LEX.SIGN + ")" const TOKEN_LITERAL_DEC : String = "(([1-9])" + LEX.DIGIT_DEC +"*)" const TOKEN_LITERAL_OCT : String = "(0" + LEX.DIGIT_OCT +"*)" const TOKEN_LITERAL_HEX : String = "(0(x|X)(" + LEX.DIGIT_HEX +")+)" const TOKEN_LITERAL_INT : String = "((\\+|\\-){0,1}" + TOKEN_LITERAL_DEC + "|" + TOKEN_LITERAL_OCT + "|" + TOKEN_LITERAL_HEX + ")" const TOKEN_LITERAL_FLOAT_DEC : String = "(" + LEX.DIGIT_DEC + "+)" const TOKEN_LITERAL_FLOAT_EXP : String = "((e|E)(\\+|\\-)?" + TOKEN_LITERAL_FLOAT_DEC + "+)" const TOKEN_LITERAL_FLOAT : String = "((\\+|\\-){0,1}(" + TOKEN_LITERAL_FLOAT_DEC + "\\." + TOKEN_LITERAL_FLOAT_DEC + "?" + TOKEN_LITERAL_FLOAT_EXP + "?)|(" + TOKEN_LITERAL_FLOAT_DEC + TOKEN_LITERAL_FLOAT_EXP + ")|(\\." + TOKEN_LITERAL_FLOAT_DEC + TOKEN_LITERAL_FLOAT_EXP + "?))" const TOKEN_SPACE : String = "(" + LEX.SPACE + ")+" const TOKEN_COMMA : String = "(" + LEX.COMMA + ")" const TOKEN_CHAR_ESC : String = "[\\\\(a|b|f|n|r|t|v|\\\\|'|\")]" const TOKEN_OCT_ESC : String = "[\\\\" + LEX.DIGIT_OCT + "{3}]" const TOKEN_HEX_ESC : String = "[\\\\(x|X)" + LEX.DIGIT_HEX + "{2}]" const TOKEN_CHAR_EXCLUDE : String = "[^\\0\\n\\\\]" const TOKEN_CHAR_VALUE : String = "(" + TOKEN_HEX_ESC + "|" + TOKEN_OCT_ESC + "|" + TOKEN_CHAR_ESC + "|" + TOKEN_CHAR_EXCLUDE + ")" const TOKEN_STRING_SINGLE : String = "('" + TOKEN_CHAR_VALUE + "*?')" const TOKEN_STRING_DOUBLE : String = "(\"" + TOKEN_CHAR_VALUE + "*?\")" const TOKEN_COMMENT_SINGLE : String = "((//[^\\n\\r]*[^\\s])|//)" const TOKEN_COMMENT_MULTI : String = "/\\*(.|[\\n\\r])*?\\*/" const TOKEN_SECOND_MESSAGE : String = "^message$" const TOKEN_SECOND_SIMPLE_DATA_TYPE : String = "^(double|float|int32|int64|uint32|uint64|sint32|sint64|fixed32|fixed64|sfixed32|sfixed64|bool|string|bytes)$" const TOKEN_SECOND_ENUM : String = "^enum$" const TOKEN_SECOND_MAP : String = "^map$" const TOKEN_SECOND_ONEOF : String = "^oneof$" const TOKEN_SECOND_LITERAL_BOOL : String = "^(true|false)$" const TOKEN_SECOND_SYNTAX : String = "^syntax$" const TOKEN_SECOND_IMPORT : String = "^import$" const TOKEN_SECOND_PACKAGE : String = "^package$" const TOKEN_SECOND_OPTION : String = "^option$" const TOKEN_SECOND_SERVICE : String = "^service$" const TOKEN_SECOND_RESERVED : String = "^reserved$" const TOKEN_SECOND_IMPORT_QUALIFICATION : String = "^(weak|public)$" const TOKEN_SECOND_FIELD_QUALIFICATION : String = "^(repeated|required|optional)$" const TOKEN_SECOND_ENUM_OPTION : String = "^allow_alias$" const TOKEN_SECOND_QUALIFICATION : String = "^(custom_option|extensions)$" const TOKEN_SECOND_FIELD_OPTION : String = "^packed$" class TokenEntrance: func _init(i : int, b : int, e : int, t : String): position = TokenPosition.new(b, e) text = t id = i var position : TokenPosition var text : String var id : int enum RANGE_STATE { INCLUDE = 0, EXCLUDE_LEFT = 1, EXCLUDE_RIGHT = 2, OVERLAY = 3, EQUAL = 4, ENTERS = 5 } class TokenRange: func _init(b : int, e : int, s): position = TokenPosition.new(b, e) state = s var position : TokenPosition var state class Token: var _regex : RegEx var _entrance : TokenEntrance = null var _entrances : Array = [] var _entrance_index : int = 0 var _id : int var _ignore : bool var _clarification : String func _init(id : int, clarification : String, regex_str : String, ignore = false): _id = id _regex = RegEx.new() _regex.compile(regex_str) _clarification = clarification _ignore = ignore func find(text : String, start : int) -> TokenEntrance: _entrance = null if !_regex.is_valid(): return null var match_result : RegExMatch = _regex.search(text, start) if match_result != null: var capture capture = match_result.get_string(0) if capture.is_empty(): return null _entrance = TokenEntrance.new(_id, match_result.get_start(0), capture.length() - 1 + match_result.get_start(0), capture) return _entrance func find_all(text : String) -> Array: var pos : int = 0 clear() while find(text, pos) != null: _entrances.append(_entrance) pos = _entrance.position.end + 1 return _entrances func add_entrance(entrance) -> void: _entrances.append(entrance) func clear() -> void: _entrance = null _entrances = [] _entrance_index = 0 func get_entrances() -> Array: return _entrances func remove_entrance(index) -> void: if index < _entrances.size(): _entrances.remove_at(index) func get_index() -> int: return _entrance_index func set_index(index : int) -> void: if index < _entrances.size(): _entrance_index = index else: _entrance_index = 0 func is_ignore() -> bool: return _ignore func get_clarification() -> String: return _clarification class TokenResult: var tokens : Array = [] var errors : Array = [] enum TOKEN_ID { UNDEFINED = -1, IDENT = 0, FULL_IDENT = 1, BRACKET_ROUND_LEFT = 2, BRACKET_ROUND_RIGHT = 3, BRACKET_CURLY_LEFT = 4, BRACKET_CURLY_RIGHT = 5, BRACKET_SQUARE_LEFT = 6, BRACKET_SQUARE_RIGHT = 7, BRACKET_ANGLE_LEFT = 8, BRACKET_ANGLE_RIGHT = 9, SEMICOLON = 10, EUQAL = 11, SIGN = 12, INT = 13, FLOAT = 14, SPACE = 15, COMMA = 16, STRING_SINGLE = 17, STRING_DOUBLE = 18, COMMENT_SINGLE = 19, COMMENT_MULTI = 20, MESSAGE = 21, SIMPLE_DATA_TYPE = 22, ENUM = 23, MAP = 24, ONEOF = 25, LITERAL_BOOL = 26, SYNTAX = 27, IMPORT = 28, PACKAGE = 29, OPTION = 30, SERVICE = 31, RESERVED = 32, IMPORT_QUALIFICATION = 33, FIELD_QUALIFICATION = 34, ENUM_OPTION = 35, QUALIFICATION = 36, FIELD_OPTION = 37, STRING = 38 } var TOKEN = { TOKEN_ID.IDENT: Token.new(TOKEN_ID.IDENT, "Identifier", TOKEN_IDENT), TOKEN_ID.FULL_IDENT: Token.new(TOKEN_ID.FULL_IDENT, "Full identifier", TOKEN_FULL_IDENT), TOKEN_ID.BRACKET_ROUND_LEFT: Token.new(TOKEN_ID.BRACKET_ROUND_LEFT, "(", TOKEN_BRACKET_ROUND_LEFT), TOKEN_ID.BRACKET_ROUND_RIGHT: Token.new(TOKEN_ID.BRACKET_ROUND_RIGHT, ")", TOKEN_BRACKET_ROUND_RIGHT), TOKEN_ID.BRACKET_CURLY_LEFT: Token.new(TOKEN_ID.BRACKET_CURLY_LEFT, "{", TOKEN_BRACKET_CURLY_LEFT), TOKEN_ID.BRACKET_CURLY_RIGHT: Token.new(TOKEN_ID.BRACKET_CURLY_RIGHT, "}", TOKEN_BRACKET_CURLY_RIGHT), TOKEN_ID.BRACKET_SQUARE_LEFT: Token.new(TOKEN_ID.BRACKET_SQUARE_LEFT, "[", TOKEN_BRACKET_SQUARE_LEFT), TOKEN_ID.BRACKET_SQUARE_RIGHT: Token.new(TOKEN_ID.BRACKET_SQUARE_RIGHT, "]", TOKEN_BRACKET_SQUARE_RIGHT), TOKEN_ID.BRACKET_ANGLE_LEFT: Token.new(TOKEN_ID.BRACKET_ANGLE_LEFT, "<", TOKEN_BRACKET_ANGLE_LEFT), TOKEN_ID.BRACKET_ANGLE_RIGHT: Token.new(TOKEN_ID.BRACKET_ANGLE_RIGHT, ">", TOKEN_BRACKET_ANGLE_RIGHT), TOKEN_ID.SEMICOLON: Token.new(TOKEN_ID.SEMICOLON, ";", TOKEN_SEMICOLON), TOKEN_ID.EUQAL: Token.new(TOKEN_ID.EUQAL, "=", TOKEN_EUQAL), TOKEN_ID.INT: Token.new(TOKEN_ID.INT, "Integer", TOKEN_LITERAL_INT), TOKEN_ID.FLOAT: Token.new(TOKEN_ID.FLOAT, "Float", TOKEN_LITERAL_FLOAT), TOKEN_ID.SPACE: Token.new(TOKEN_ID.SPACE, "Space", TOKEN_SPACE), TOKEN_ID.COMMA: Token.new(TOKEN_ID.COMMA, ",", TOKEN_COMMA), TOKEN_ID.STRING_SINGLE: Token.new(TOKEN_ID.STRING_SINGLE, "'String'", TOKEN_STRING_SINGLE), TOKEN_ID.STRING_DOUBLE: Token.new(TOKEN_ID.STRING_DOUBLE, "\"String\"", TOKEN_STRING_DOUBLE), TOKEN_ID.COMMENT_SINGLE: Token.new(TOKEN_ID.COMMENT_SINGLE, "//Comment", TOKEN_COMMENT_SINGLE), TOKEN_ID.COMMENT_MULTI: Token.new(TOKEN_ID.COMMENT_MULTI, "/*Comment*/", TOKEN_COMMENT_MULTI), TOKEN_ID.MESSAGE: Token.new(TOKEN_ID.MESSAGE, "Message", TOKEN_SECOND_MESSAGE, true), TOKEN_ID.SIMPLE_DATA_TYPE: Token.new(TOKEN_ID.SIMPLE_DATA_TYPE, "Data type", TOKEN_SECOND_SIMPLE_DATA_TYPE, true), TOKEN_ID.ENUM: Token.new(TOKEN_ID.ENUM, "Enum", TOKEN_SECOND_ENUM, true), TOKEN_ID.MAP: Token.new(TOKEN_ID.MAP, "Map", TOKEN_SECOND_MAP, true), TOKEN_ID.ONEOF: Token.new(TOKEN_ID.ONEOF, "OneOf", TOKEN_SECOND_ONEOF, true), TOKEN_ID.LITERAL_BOOL: Token.new(TOKEN_ID.LITERAL_BOOL, "Bool literal", TOKEN_SECOND_LITERAL_BOOL, true), TOKEN_ID.SYNTAX: Token.new(TOKEN_ID.SYNTAX, "Syntax", TOKEN_SECOND_SYNTAX, true), TOKEN_ID.IMPORT: Token.new(TOKEN_ID.IMPORT, "Import", TOKEN_SECOND_IMPORT, true), TOKEN_ID.PACKAGE: Token.new(TOKEN_ID.PACKAGE, "Package", TOKEN_SECOND_PACKAGE, true), TOKEN_ID.OPTION: Token.new(TOKEN_ID.OPTION, "Option", TOKEN_SECOND_OPTION, true), TOKEN_ID.SERVICE: Token.new(TOKEN_ID.SERVICE, "Service", TOKEN_SECOND_SERVICE, true), TOKEN_ID.RESERVED: Token.new(TOKEN_ID.RESERVED, "Reserved", TOKEN_SECOND_RESERVED, true), TOKEN_ID.IMPORT_QUALIFICATION: Token.new(TOKEN_ID.IMPORT_QUALIFICATION, "Import qualification", TOKEN_SECOND_IMPORT_QUALIFICATION, true), TOKEN_ID.FIELD_QUALIFICATION: Token.new(TOKEN_ID.FIELD_QUALIFICATION, "Field qualification", TOKEN_SECOND_FIELD_QUALIFICATION, true), TOKEN_ID.ENUM_OPTION: Token.new(TOKEN_ID.ENUM_OPTION, "Enum option", TOKEN_SECOND_ENUM_OPTION, true), TOKEN_ID.QUALIFICATION: Token.new(TOKEN_ID.QUALIFICATION, "Qualification", TOKEN_SECOND_QUALIFICATION, true), TOKEN_ID.FIELD_OPTION: Token.new(TOKEN_ID.FIELD_OPTION, "Field option", TOKEN_SECOND_FIELD_OPTION, true), TOKEN_ID.STRING: Token.new(TOKEN_ID.STRING, "String", "", true) } static func check_range(main : TokenEntrance, current : TokenEntrance) -> TokenRange: if main.position.begin > current.position.begin: if main.position.end > current.position.end: if main.position.begin >= current.position.end: return TokenRange.new(current.position.begin, current.position.end, RANGE_STATE.EXCLUDE_LEFT) else: return TokenRange.new(main.position.begin, current.position.end, RANGE_STATE.OVERLAY) else: return TokenRange.new(current.position.begin, current.position.end, RANGE_STATE.ENTERS) elif main.position.begin < current.position.begin: if main.position.end >= current.position.end: return TokenRange.new(main.position.begin, main.position.end, RANGE_STATE.INCLUDE) else: if main.position.end < current.position.begin: return TokenRange.new(main.position.begin, main.position.end, RANGE_STATE.EXCLUDE_RIGHT) else: return TokenRange.new(main.position.begin, current.position.end, RANGE_STATE.OVERLAY) else: if main.position.end == current.position.end: return TokenRange.new(main.position.begin, main.position.end, RANGE_STATE.EQUAL) elif main.position.end > current.position.end: return TokenRange.new(main.position.begin, main.position.end, RANGE_STATE.INCLUDE) else: return TokenRange.new(current.position.begin, current.position.end, RANGE_STATE.ENTERS) func tokenizer() -> TokenResult: for k in TOKEN: if !TOKEN[k].is_ignore(): TOKEN[k].find_all(document.text) var second_tokens : Array = [] second_tokens.append(TOKEN[TOKEN_ID.MESSAGE]) second_tokens.append(TOKEN[TOKEN_ID.SIMPLE_DATA_TYPE]) second_tokens.append(TOKEN[TOKEN_ID.ENUM]) second_tokens.append(TOKEN[TOKEN_ID.MAP]) second_tokens.append(TOKEN[TOKEN_ID.ONEOF]) second_tokens.append(TOKEN[TOKEN_ID.LITERAL_BOOL]) second_tokens.append(TOKEN[TOKEN_ID.SYNTAX]) second_tokens.append(TOKEN[TOKEN_ID.IMPORT]) second_tokens.append(TOKEN[TOKEN_ID.PACKAGE]) second_tokens.append(TOKEN[TOKEN_ID.OPTION]) second_tokens.append(TOKEN[TOKEN_ID.SERVICE]) second_tokens.append(TOKEN[TOKEN_ID.RESERVED]) second_tokens.append(TOKEN[TOKEN_ID.IMPORT_QUALIFICATION]) second_tokens.append(TOKEN[TOKEN_ID.FIELD_QUALIFICATION]) second_tokens.append(TOKEN[TOKEN_ID.ENUM_OPTION]) second_tokens.append(TOKEN[TOKEN_ID.QUALIFICATION]) second_tokens.append(TOKEN[TOKEN_ID.FIELD_OPTION]) var ident_token : Token = TOKEN[TOKEN_ID.IDENT] for sec_token in second_tokens: var remove_indexes : Array = [] for i in range(ident_token.get_entrances().size()): var entrance : TokenEntrance = sec_token.find(ident_token.get_entrances()[i].text, 0) if entrance != null: entrance.position.begin = ident_token.get_entrances()[i].position.begin entrance.position.end = ident_token.get_entrances()[i].position.end sec_token.add_entrance(entrance) remove_indexes.append(i) for i in range(remove_indexes.size()): ident_token.remove_entrance(remove_indexes[i] - i) for v in TOKEN[TOKEN_ID.STRING_DOUBLE].get_entrances(): v.id = TOKEN_ID.STRING TOKEN[TOKEN_ID.STRING].add_entrance(v) TOKEN[TOKEN_ID.STRING_DOUBLE].clear() for v in TOKEN[TOKEN_ID.STRING_SINGLE].get_entrances(): v.id = TOKEN_ID.STRING TOKEN[TOKEN_ID.STRING].add_entrance(v) TOKEN[TOKEN_ID.STRING_SINGLE].clear() var main_token : TokenEntrance var cur_token : TokenEntrance var main_index : int = -1 var token_index_flag : bool = false var result : TokenResult = TokenResult.new() var check : TokenRange var end : bool = false var all : bool = false var repeat : bool = false while true: all = true for k in TOKEN: if main_index == k: continue repeat = false while TOKEN[k].get_entrances().size() > 0: all = false if !token_index_flag: main_index = k main_token = TOKEN[main_index].get_entrances()[0] token_index_flag = true break else: cur_token = TOKEN[k].get_entrances()[0] check = check_range(main_token, cur_token) if check.state == RANGE_STATE.INCLUDE: TOKEN[k].remove_entrance(0) end = true elif check.state == RANGE_STATE.EXCLUDE_LEFT: main_token = cur_token main_index = k end = false repeat = true break elif check.state == RANGE_STATE.EXCLUDE_RIGHT: end = true break elif check.state == RANGE_STATE.OVERLAY || check.state == RANGE_STATE.EQUAL: result.errors.append(check) TOKEN[main_index].remove_entrance(0) TOKEN[k].remove_entrance(0) token_index_flag = false end = false repeat = true break elif check.state == RANGE_STATE.ENTERS: TOKEN[main_index].remove_entrance(0) main_token = cur_token main_index = k end = false repeat = true break if repeat: break if end: if TOKEN[main_index].get_entrances().size() > 0: result.tokens.append(main_token) TOKEN[main_index].remove_entrance(0) token_index_flag = false if all: break return result static func check_tokens_integrity(tokens : Array, end : int) -> Array: var cur_index : int = 0 var result : Array = [] for v in tokens: if v.position.begin > cur_index: result.append(TokenPosition.new(cur_index, v.position.begin)) cur_index = v.position.end + 1 if cur_index < end: result.append(TokenPosition.new(cur_index, end)) return result static func comment_space_processing(tokens : Array) -> void: var remove_indexes : Array = [] for i in range(tokens.size()): if tokens[i].id == TOKEN_ID.COMMENT_SINGLE || tokens[i].id == TOKEN_ID.COMMENT_MULTI: tokens[i].id = TOKEN_ID.SPACE var space_index : int = -1 for i in range(tokens.size()): if tokens[i].id == TOKEN_ID.SPACE: if space_index >= 0: tokens[space_index].position.end = tokens[i].position.end tokens[space_index].text = tokens[space_index].text + tokens[i].text remove_indexes.append(i) else: space_index = i else: space_index = -1 for i in range(remove_indexes.size()): tokens.remove_at(remove_indexes[i] - i) #Analysis rule enum AR { MAYBE = 1, MUST_ONE = 2, ANY = 3, OR = 4, MAYBE_BEGIN = 5, MAYBE_END = 6, ANY_BEGIN = 7, ANY_END = 8 } #Space rule (space after token) enum SP { MAYBE = 1, MUST = 2, NO = 3 } #Analysis Syntax Description class ASD: func _init(t, s : int = SP.MAYBE, r : int = AR.MUST_ONE, i : bool = false): token = t space = s rule = r importance = i var token var space : int var rule : int var importance : bool var TEMPLATE_SYNTAX : Array = [ Callable(self, "desc_syntax"), ASD.new(TOKEN_ID.SYNTAX), ASD.new(TOKEN_ID.EUQAL), ASD.new(TOKEN_ID.STRING, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.SEMICOLON) ] var TEMPLATE_IMPORT : Array = [ Callable(self, "desc_import"), ASD.new(TOKEN_ID.IMPORT, SP.MUST), ASD.new(TOKEN_ID.IMPORT_QUALIFICATION, SP.MUST, AR.MAYBE, true), ASD.new(TOKEN_ID.STRING, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.SEMICOLON) ] var TEMPLATE_PACKAGE : Array = [ Callable(self, "desc_package"), ASD.new(TOKEN_ID.PACKAGE, SP.MUST), ASD.new([TOKEN_ID.IDENT, TOKEN_ID.FULL_IDENT], SP.MAYBE, AR.OR, true), ASD.new(TOKEN_ID.SEMICOLON) ] var TEMPLATE_OPTION : Array = [ Callable(self, "desc_option"), ASD.new(TOKEN_ID.OPTION, SP.MUST), ASD.new([TOKEN_ID.IDENT, TOKEN_ID.FULL_IDENT], SP.MAYBE, AR.OR, true), ASD.new(TOKEN_ID.EUQAL), ASD.new([TOKEN_ID.STRING, TOKEN_ID.INT, TOKEN_ID.FLOAT, TOKEN_ID.LITERAL_BOOL], SP.MAYBE, AR.OR, true), ASD.new(TOKEN_ID.SEMICOLON) ] var TEMPLATE_FIELD : Array = [ Callable(self, "desc_field"), ASD.new(TOKEN_ID.FIELD_QUALIFICATION, SP.MUST, AR.MAYBE, true), ASD.new([TOKEN_ID.SIMPLE_DATA_TYPE, TOKEN_ID.IDENT, TOKEN_ID.FULL_IDENT], SP.MAYBE, AR.OR, true), ASD.new(TOKEN_ID.IDENT, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.EUQAL), ASD.new(TOKEN_ID.INT, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.BRACKET_SQUARE_LEFT, SP.MAYBE, AR.MAYBE_BEGIN), ASD.new(TOKEN_ID.FIELD_OPTION, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.EUQAL), ASD.new(TOKEN_ID.LITERAL_BOOL, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.BRACKET_SQUARE_RIGHT, SP.MAYBE, AR.MAYBE_END), ASD.new(TOKEN_ID.SEMICOLON) ] var TEMPLATE_FIELD_ONEOF : Array = TEMPLATE_FIELD var TEMPLATE_MAP_FIELD : Array = [ Callable(self, "desc_map_field"), ASD.new(TOKEN_ID.MAP), ASD.new(TOKEN_ID.BRACKET_ANGLE_LEFT), ASD.new(TOKEN_ID.SIMPLE_DATA_TYPE, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.COMMA), ASD.new([TOKEN_ID.SIMPLE_DATA_TYPE, TOKEN_ID.IDENT, TOKEN_ID.FULL_IDENT], SP.MAYBE, AR.OR, true), ASD.new(TOKEN_ID.BRACKET_ANGLE_RIGHT, SP.MUST), ASD.new(TOKEN_ID.IDENT, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.EUQAL), ASD.new(TOKEN_ID.INT, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.BRACKET_SQUARE_LEFT, SP.MAYBE, AR.MAYBE_BEGIN), ASD.new(TOKEN_ID.FIELD_OPTION, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.EUQAL), ASD.new(TOKEN_ID.LITERAL_BOOL, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.BRACKET_SQUARE_RIGHT, SP.MAYBE, AR.MAYBE_END), ASD.new(TOKEN_ID.SEMICOLON) ] var TEMPLATE_MAP_FIELD_ONEOF : Array = TEMPLATE_MAP_FIELD var TEMPLATE_ENUM : Array = [ Callable(self, "desc_enum"), ASD.new(TOKEN_ID.ENUM, SP.MUST), ASD.new(TOKEN_ID.IDENT, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.BRACKET_CURLY_LEFT), ASD.new(TOKEN_ID.OPTION, SP.MUST, AR.MAYBE_BEGIN), ASD.new(TOKEN_ID.ENUM_OPTION, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.EUQAL), ASD.new(TOKEN_ID.LITERAL_BOOL, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.SEMICOLON, SP.MAYBE, AR.MAYBE_END), ASD.new(TOKEN_ID.IDENT, SP.MAYBE, AR.ANY_BEGIN, true), ASD.new(TOKEN_ID.EUQAL), ASD.new(TOKEN_ID.INT, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.SEMICOLON, SP.MAYBE, AR.ANY_END), ASD.new(TOKEN_ID.BRACKET_CURLY_RIGHT) ] var TEMPLATE_MESSAGE_HEAD : Array = [ Callable(self, "desc_message_head"), ASD.new(TOKEN_ID.MESSAGE, SP.MUST), ASD.new(TOKEN_ID.IDENT, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.BRACKET_CURLY_LEFT) ] var TEMPLATE_MESSAGE_TAIL : Array = [ Callable(self, "desc_message_tail"), ASD.new(TOKEN_ID.BRACKET_CURLY_RIGHT) ] var TEMPLATE_ONEOF_HEAD : Array = [ Callable(self, "desc_oneof_head"), ASD.new(TOKEN_ID.ONEOF, SP.MUST), ASD.new(TOKEN_ID.IDENT, SP.MAYBE, AR.MUST_ONE, true), ASD.new(TOKEN_ID.BRACKET_CURLY_LEFT), ] var TEMPLATE_ONEOF_TAIL : Array = [ Callable(self, "desc_oneof_tail"), ASD.new(TOKEN_ID.BRACKET_CURLY_RIGHT) ] var TEMPLATE_BEGIN : Array = [ null, ASD.new(TOKEN_ID.SPACE, SP.NO, AR.MAYBE) ] var TEMPLATE_END : Array = [ null ] func get_token_id(tokens : Array, index : int) -> int: if index < tokens.size(): return tokens[index].id return TOKEN_ID.UNDEFINED enum COMPARE_STATE { DONE = 0, MISMATCH = 1, INCOMPLETE = 2, ERROR_VALUE = 3 } class TokenCompare: func _init(s : int, i : int, d : String = ""): state = s index = i description = d var state : int var index : int var description : String func check_space(tokens : Array, index : int, space) -> int: if get_token_id(tokens, index) == TOKEN_ID.SPACE: if space == SP.MAYBE: return 1 elif space == SP.MUST: return 1 elif space == SP.NO: return -1 else: if space == SP.MUST: return -2 return 0 class IndexedToken: func _init(t : TokenEntrance, i : int): token = t index = i var token : TokenEntrance var index : int func token_importance_checkadd(template : ASD, token : TokenEntrance, index : int, importance : Array) -> void: if template.importance: importance.append(IndexedToken.new(token, index)) class CompareSettings: func _init(ci : int, n : int, pi : int, pn : String = ""): construction_index = ci nesting = n parent_index = pi parent_name = pn var construction_index : int var nesting : int var parent_index : int var parent_name : String func description_compare(template : Array, tokens : Array, index : int, settings : CompareSettings) -> TokenCompare: var j : int = index var space : int var rule : int var rule_flag : bool var cont : bool var check : int var maybe_group_skip : bool = false var any_group_index : int = -1 var any_end_group_index : int = -1 var i : int = 0 var importance : Array = [] while true: i += 1 if i >= template.size(): break rule_flag = false cont = false rule = template[i].rule space = template[i].space if rule == AR.MAYBE_END && maybe_group_skip: maybe_group_skip = false continue if maybe_group_skip: continue if rule == AR.MAYBE: if template[i].token == get_token_id(tokens, j): token_importance_checkadd(template[i], tokens[j], j, importance) rule_flag = true else: continue elif rule == AR.MUST_ONE || rule == AR.MAYBE_END || rule == AR.ANY_END: if template[i].token == get_token_id(tokens, j): token_importance_checkadd(template[i], tokens[j], j, importance) rule_flag = true elif rule == AR.ANY: var find_any : bool = false while true: if template[i].token == get_token_id(tokens, j): token_importance_checkadd(template[i], tokens[j], j, importance) find_any = true j += 1 check = check_space(tokens, j, space) if check < 0: return TokenCompare.new(COMPARE_STATE.INCOMPLETE, j) else: j += check else: if find_any: cont = true break elif rule == AR.OR: var or_tokens = template[i].token for v in or_tokens: if v == get_token_id(tokens, j): token_importance_checkadd(template[i], tokens[j], j, importance) j += 1 check = check_space(tokens, j, space) if check < 0: return TokenCompare.new(COMPARE_STATE.INCOMPLETE, j) else: j += check cont = true break elif rule == AR.MAYBE_BEGIN: if template[i].token == get_token_id(tokens, j): token_importance_checkadd(template[i], tokens[j], j, importance) rule_flag = true else: maybe_group_skip = true continue elif rule == AR.ANY_BEGIN: if template[i].token == get_token_id(tokens, j): token_importance_checkadd(template[i], tokens[j], j, importance) rule_flag = true any_group_index = i else: if any_end_group_index > 0: any_group_index = -1 i = any_end_group_index any_end_group_index = -1 continue if cont: continue if rule_flag: j += 1 check = check_space(tokens, j, space) if check < 0: return TokenCompare.new(COMPARE_STATE.INCOMPLETE, j) else: j += check else: if j > index: return TokenCompare.new(COMPARE_STATE.INCOMPLETE, j) else: return TokenCompare.new(COMPARE_STATE.MISMATCH, j) if any_group_index >= 0 && rule == AR.ANY_END: any_end_group_index = i i = any_group_index - 1 if template[0] != null: var result : DescriptionResult = template[0].call(importance, settings) if !result.success: return TokenCompare.new(COMPARE_STATE.ERROR_VALUE, result.error, result.description) return TokenCompare.new(COMPARE_STATE.DONE, j) var DESCRIPTION : Array = [ TEMPLATE_BEGIN, #0 TEMPLATE_SYNTAX, #1 TEMPLATE_IMPORT, #2 TEMPLATE_PACKAGE, #3 TEMPLATE_OPTION, #4 TEMPLATE_FIELD, #5 TEMPLATE_FIELD_ONEOF, #6 TEMPLATE_MAP_FIELD, #7 TEMPLATE_MAP_FIELD_ONEOF, #8 TEMPLATE_ENUM, #9 TEMPLATE_MESSAGE_HEAD, #10 TEMPLATE_MESSAGE_TAIL, #11 TEMPLATE_ONEOF_HEAD, #12 TEMPLATE_ONEOF_TAIL, #13 TEMPLATE_END #14 ] enum JUMP { NOTHING = 0, #nothing SIMPLE = 1, #simple jump NESTED_INCREMENT = 2, #nested increment NESTED_DECREMENT = 3, #nested decrement MUST_NESTED_SIMPLE = 4, #check: must be nested > 0 MUST_NESTED_INCREMENT = 5, #check: must be nested > 0, then nested increment MUST_NESTED_DECREMENT = 6, #nested decrement, then check: must be nested > 0 } var TRANSLATION_TABLE : Array = [ # BEGIN SYNTAX IMPORT PACKAGE OPTION FIELD FIELD_O MAP_F MAP_F_O ENUM MES_H MES_T ONEOF_H ONEOF_T END [ 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], #BEGIN [ 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 1], #SYNTAX [ 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 1], #IMPORT [ 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 1], #PACKAGE [ 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 1], #OPTION [ 0, 0, 0, 0, 0, 4, 0, 4, 0, 1, 2, 3, 5, 0, 0], #FIELD [ 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 6, 0], #FIELD_ONEOF [ 0, 0, 0, 0, 0, 4, 0, 4, 0, 1, 2, 3, 5, 0, 0], #MAP_F [ 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 6, 0], #MAP_F_ONEOF [ 0, 0, 0, 0, 0, 4, 0, 4, 0, 1, 2, 3, 5, 0, 1], #ENUM [ 0, 0, 0, 0, 0, 4, 0, 4, 0, 1, 2, 3, 5, 0, 0], #MES_H [ 0, 0, 0, 0, 0, 4, 0, 4, 0, 1, 2, 3, 5, 0, 1], #MES_T [ 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0], #ONEOF_H [ 0, 0, 0, 0, 0, 4, 0, 4, 0, 1, 2, 3, 5, 0, 1], #ONEOF_T [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] #END ] class Construction: func _init(b : int, e : int, d : int): begin_token_index = b end_token_index = e description = d var begin_token_index : int var end_token_index : int var description : int class TranslationResult: var constructions : Array = [] var done : bool = false var error_description_id : int = -1 var error_description_text : String = "" var parse_token_index : int = 0 var error_token_index : int = 0 func analyze_tokens(tokens : Array) -> TranslationResult: var i : int = 0 var result : TranslationResult = TranslationResult.new() var comp : TokenCompare var cur_template_id : int = 0 var error : bool = false var template_index : int var comp_set : CompareSettings = CompareSettings.new(result.constructions.size(), 0, -1) comp = description_compare(DESCRIPTION[cur_template_id], tokens, i, comp_set) if comp.state == COMPARE_STATE.DONE: i = comp.index while true: var end : bool = true var find : bool = false for j in range(TRANSLATION_TABLE[cur_template_id].size()): template_index = j if j == DESCRIPTION.size() - 1 && i < tokens.size(): end = false if result.error_description_id < 0: error = true break if TRANSLATION_TABLE[cur_template_id][j] > 0: end = false comp_set.construction_index = result.constructions.size() comp = description_compare(DESCRIPTION[j], tokens, i, comp_set) if comp.state == COMPARE_STATE.DONE: if TRANSLATION_TABLE[cur_template_id][j] == JUMP.NESTED_INCREMENT: comp_set.nesting += 1 elif TRANSLATION_TABLE[cur_template_id][j] == JUMP.NESTED_DECREMENT: comp_set.nesting -= 1 if comp_set.nesting < 0: error = true break elif TRANSLATION_TABLE[cur_template_id][j] == JUMP.MUST_NESTED_SIMPLE: if comp_set.nesting <= 0: error = true break elif TRANSLATION_TABLE[cur_template_id][j] == JUMP.MUST_NESTED_INCREMENT: if comp_set.nesting <= 0: error = true break comp_set.nesting += 1 elif TRANSLATION_TABLE[cur_template_id][j] == JUMP.MUST_NESTED_DECREMENT: comp_set.nesting -= 1 if comp_set.nesting <= 0: error = true break result.constructions.append(Construction.new(i, comp.index, j)) find = true i = comp.index cur_template_id = j if i == tokens.size(): if TRANSLATION_TABLE[cur_template_id][DESCRIPTION.size() - 1] == JUMP.SIMPLE: if comp_set.nesting == 0: end = true else: error = true else: error = true elif i > tokens.size(): error = true break elif comp.state == COMPARE_STATE.INCOMPLETE: error = true break elif comp.state == COMPARE_STATE.ERROR_VALUE: error = true break if error: result.error_description_text = comp.description result.error_description_id = template_index result.parse_token_index = i if comp.index >= tokens.size(): result.error_token_index = tokens.size() - 1 else: result.error_token_index = comp.index if end: result.done = true result.error_description_id = -1 break if !find: break return result enum CLASS_TYPE { ENUM = 0, MESSAGE = 1, MAP = 2 } enum FIELD_TYPE { UNDEFINED = -1, INT32 = 0, SINT32 = 1, UINT32 = 2, INT64 = 3, SINT64 = 4, UINT64 = 5, BOOL = 6, ENUM = 7, FIXED32 = 8, SFIXED32 = 9, FLOAT = 10, FIXED64 = 11, SFIXED64 = 12, DOUBLE = 13, STRING = 14, BYTES = 15, MESSAGE = 16, MAP = 17 } enum FIELD_QUALIFICATOR { OPTIONAL = 0, REQUIRED = 1, REPEATED = 2, RESERVED = 3 } enum FIELD_OPTION { PACKED = 0, NOT_PACKED = 1 } class ASTClass: func _init(n : String, t : int, p : int, pn : String, o : String, ci : int): name = n type = t parent_index = p parent_name = pn option = o construction_index = ci values = [] var name : String var type : int var parent_index : int var parent_name : String var option : String var construction_index var values : Array func copy() -> ASTClass: var res : ASTClass = ASTClass.new(name, type, parent_index, parent_name, option, construction_index) for v in values: res.values.append(v.copy()) return res class ASTEnumValue: func _init(n : String, v : String): name = n value = v var name : String var value : String func copy() -> ASTEnumValue: return ASTEnumValue.new(name, value) class ASTField: func _init(t, n : String, tn : String, p : int, q : int, o : int, ci : int, mf : bool): tag = t name = n type_name = tn parent_class_id = p qualificator = q option = o construction_index = ci is_map_field = mf var tag var name : String var type_name : String var parent_class_id : int var qualificator : int var option : int var construction_index : int var is_map_field : bool var field_type : int = FIELD_TYPE.UNDEFINED var type_class_id : int = -1 func copy() -> ASTField: var res : ASTField = ASTField.new(tag, name, type_name, parent_class_id, qualificator, option, construction_index, is_map_field) res.field_type = field_type res.type_class_id = type_class_id return res enum AST_GROUP_RULE { ONEOF = 0, ALL = 1 } class ASTFieldGroup: func _init(n : String, pi : int, r : int): name = n parent_class_id = pi rule = r opened = true var name : String var parent_class_id : int var rule : int var field_indexes : Array = [] var opened : bool func copy() -> ASTFieldGroup: var res : ASTFieldGroup = ASTFieldGroup.new(name, parent_class_id, rule) res.opened = opened for fi in field_indexes: res.field_indexes.append(fi) return res class ASTImport: func _init(a_path : String, a_public : bool, sha : String): path = a_path public = a_public sha256 = sha var path : String var public : bool var sha256 : String var class_table : Array = [] var field_table : Array = [] var group_table : Array = [] var import_table : Array = [] var proto_version : int = 0 class DescriptionResult: func _init(s : bool = true, e = null, d : String = ""): success = s error = e description = d var success : bool var error var description : String static func get_text_from_token(string_token : TokenEntrance) -> String: return string_token.text.substr(1, string_token.text.length() - 2) func desc_syntax(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: var result : DescriptionResult = DescriptionResult.new() var s : String = get_text_from_token(indexed_tokens[0].token) if s == "proto2": proto_version = 2 elif s == "proto3": proto_version = 3 else: result.success = false result.error = indexed_tokens[0].index result.description = "Unspecified version of the protocol. Use \"proto2\" or \"proto3\" syntax string." return result func desc_import(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: var result : DescriptionResult = DescriptionResult.new() var offset : int = 0 var public : bool = false if indexed_tokens[offset].token.id == TOKEN_ID.IMPORT_QUALIFICATION: if indexed_tokens[offset].token.text == "public": public = true offset += 1 var f_name : String = path_dir + get_text_from_token(indexed_tokens[offset].token) var sha : String = FileAccess.get_sha256(f_name) if FileAccess.file_exists(f_name): for i in import_table: if i.path == f_name: result.success = false result.error = indexed_tokens[offset].index result.description = "File '" + f_name + "' already imported." return result if i.sha256 == sha: result.success = false result.error = indexed_tokens[offset].index result.description = "File '" + f_name + "' with matching SHA256 already imported." return result import_table.append(ASTImport.new(f_name, public, sha)) else: result.success = false result.error = indexed_tokens[offset].index result.description = "Import file '" + f_name + "' not found." return result func desc_package(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: printerr("UNRELEASED desc_package: ", indexed_tokens.size(), ", nesting: ", settings.nesting) var result : DescriptionResult = DescriptionResult.new() return result func desc_option(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: printerr("UNRELEASED desc_option: ", indexed_tokens.size(), ", nesting: ", settings.nesting) var result : DescriptionResult = DescriptionResult.new() return result func desc_field(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: var result : DescriptionResult = DescriptionResult.new() var qualifcator : int = FIELD_QUALIFICATOR.OPTIONAL var option : int var offset : int = 0 if proto_version == 3: option = FIELD_OPTION.PACKED if indexed_tokens[offset].token.id == TOKEN_ID.FIELD_QUALIFICATION: if indexed_tokens[offset].token.text == "repeated": qualifcator = FIELD_QUALIFICATOR.REPEATED elif indexed_tokens[offset].token.text == "required" || indexed_tokens[offset].token.text == "optional": result.success = false result.error = indexed_tokens[offset].index result.description = "Using the 'required' or 'optional' qualificator is unacceptable in Protobuf v3." return result offset += 1 if proto_version == 2: option = FIELD_OPTION.NOT_PACKED if !(group_table.size() > 0 && group_table[group_table.size() - 1].opened): if indexed_tokens[offset].token.id == TOKEN_ID.FIELD_QUALIFICATION: if indexed_tokens[offset].token.text == "repeated": qualifcator = FIELD_QUALIFICATOR.REPEATED elif indexed_tokens[offset].token.text == "required": qualifcator = FIELD_QUALIFICATOR.REQUIRED elif indexed_tokens[offset].token.text == "optional": qualifcator = FIELD_QUALIFICATOR.OPTIONAL offset += 1 else: if class_table[settings.parent_index].type == CLASS_TYPE.MESSAGE: result.success = false result.error = indexed_tokens[offset].index result.description = "Using the 'required', 'optional' or 'repeated' qualificator necessarily in Protobuf v2." return result var type_name : String = indexed_tokens[offset].token.text; offset += 1 var field_name : String = indexed_tokens[offset].token.text; offset += 1 var tag : String = indexed_tokens[offset].token.text; offset += 1 if indexed_tokens.size() == offset + 2: if indexed_tokens[offset].token.text == "packed": offset += 1 if indexed_tokens[offset].token.text == "true": option = FIELD_OPTION.PACKED else: option = FIELD_OPTION.NOT_PACKED else: result.success = false result.error = indexed_tokens[offset].index result.description = "Undefined field option." return result if group_table.size() > 0: if group_table[group_table.size() - 1].opened: if indexed_tokens[0].token.id == TOKEN_ID.FIELD_QUALIFICATION: result.success = false result.error = indexed_tokens[0].index result.description = "Using the 'required', 'optional' or 'repeated' qualificator is unacceptable in 'OneOf' field." return result group_table[group_table.size() - 1].field_indexes.append(field_table.size()) field_table.append(ASTField.new(tag, field_name, type_name, settings.parent_index, qualifcator, option, settings.construction_index, false)) return result func desc_map_field(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: var result : DescriptionResult = DescriptionResult.new() var qualifcator : int = FIELD_QUALIFICATOR.REPEATED var option : int var offset : int = 0 if proto_version == 3: option = FIELD_OPTION.PACKED if proto_version == 2: option = FIELD_OPTION.NOT_PACKED var key_type_name : String = indexed_tokens[offset].token.text; offset += 1 if key_type_name == "float" || key_type_name == "double" || key_type_name == "bytes": result.success = false result.error = indexed_tokens[offset - 1].index result.description = "Map 'key_type' can't be floating point types and bytes." var type_name : String = indexed_tokens[offset].token.text; offset += 1 var field_name : String = indexed_tokens[offset].token.text; offset += 1 var tag : String = indexed_tokens[offset].token.text; offset += 1 if indexed_tokens.size() == offset + 2: if indexed_tokens[offset].token.text == "packed": offset += 1 if indexed_tokens[offset] == "true": option = FIELD_OPTION.PACKED else: option = FIELD_OPTION.NOT_PACKED else: result.success = false result.error = indexed_tokens[offset].index result.description = "Undefined field option." if group_table.size() > 0: if group_table[group_table.size() - 1].opened: group_table[group_table.size() - 1].field_indexes.append(field_table.size()) class_table.append(ASTClass.new("map_type_" + field_name, CLASS_TYPE.MAP, settings.parent_index, settings.parent_name, "", settings.construction_index)) field_table.append(ASTField.new(tag, field_name, "map_type_" + field_name, settings.parent_index, qualifcator, option, settings.construction_index, false)) field_table.append(ASTField.new(1, "key", key_type_name, class_table.size() - 1, FIELD_QUALIFICATOR.OPTIONAL, option, settings.construction_index, true)) field_table.append(ASTField.new(2, "value", type_name, class_table.size() - 1, FIELD_QUALIFICATOR.OPTIONAL, option, settings.construction_index, true)) return result func desc_enum(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: var result : DescriptionResult = DescriptionResult.new() var option : String = "" var offset : int = 0 var type_name : String = indexed_tokens[offset].token.text; offset += 1 if indexed_tokens[offset].token.id == TOKEN_ID.ENUM_OPTION: if indexed_tokens[offset].token.text == "allow_alias" && indexed_tokens[offset + 1].token.text == "true": option = "allow_alias" offset += 2 var value : ASTEnumValue var enum_class : ASTClass = ASTClass.new(type_name, CLASS_TYPE.ENUM, settings.parent_index, settings.parent_name, option, settings.construction_index) var first_value : bool = true while offset < indexed_tokens.size(): if first_value: if indexed_tokens[offset + 1].token.text != "0": result.success = false result.error = indexed_tokens[offset + 1].index result.description = "For Enums, the default value is the first defined enum value, which must be 0." break first_value = false #if indexed_tokens[offset + 1].token.text[0] == "+" || indexed_tokens[offset + 1].token.text[0] == "-": # result.success = false # result.error = indexed_tokens[offset + 1].index # result.description = "For Enums, signed values are not allowed." # break value = ASTEnumValue.new(indexed_tokens[offset].token.text, indexed_tokens[offset + 1].token.text) enum_class.values.append(value) offset += 2 class_table.append(enum_class) return result func desc_message_head(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: var result : DescriptionResult = DescriptionResult.new() class_table.append(ASTClass.new(indexed_tokens[0].token.text, CLASS_TYPE.MESSAGE, settings.parent_index, settings.parent_name, "", settings.construction_index)) settings.parent_index = class_table.size() - 1 settings.parent_name = settings.parent_name + "." + indexed_tokens[0].token.text return result func desc_message_tail(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: settings.parent_index = class_table[settings.parent_index].parent_index settings.parent_name = class_table[settings.parent_index + 1].parent_name var result : DescriptionResult = DescriptionResult.new() return result func desc_oneof_head(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: var result : DescriptionResult = DescriptionResult.new() for g in group_table: if g.parent_class_id == settings.parent_index && g.name == indexed_tokens[0].token.text: result.success = false result.error = indexed_tokens[0].index result.description = "OneOf name must be unique." return result group_table.append(ASTFieldGroup.new(indexed_tokens[0].token.text, settings.parent_index, AST_GROUP_RULE.ONEOF)) return result func desc_oneof_tail(indexed_tokens : Array, settings : CompareSettings) -> DescriptionResult: group_table[group_table.size() - 1].opened = false var result : DescriptionResult = DescriptionResult.new() return result func analyze() -> AnalyzeResult: var analyze_result : AnalyzeResult = AnalyzeResult.new() analyze_result.doc = document analyze_result.classes = class_table analyze_result.fields = field_table analyze_result.groups = group_table analyze_result.state = false var result : TokenResult = tokenizer() if result.errors.size() > 0: for v in result.errors: var spos : Helper.StringPosition = Helper.str_pos(document.text, v.position) var err_text : String = "Unexpected token intersection " + "'" + document.text.substr(v.position.begin, spos.length) + "'" printerr(Helper.error_string(document.name, spos.str_num, spos.column, err_text)) else: var integrity = check_tokens_integrity(result.tokens, document.text.length() - 1) if integrity.size() > 0: for v in integrity: var spos: Helper.StringPosition = Helper.str_pos(document.text, TokenPosition.new(v.begin, v.end)) var err_text : String = "Unexpected token " + "'" + document.text.substr(v.begin, spos.length) + "'" printerr(Helper.error_string(document.name, spos.str_num, spos.column, err_text)) else: analyze_result.tokens = result.tokens comment_space_processing(result.tokens) var syntax : TranslationResult = analyze_tokens(result.tokens) if !syntax.done: var pos_main : TokenPosition = Helper.text_pos(result.tokens, syntax.parse_token_index) var pos_inner : TokenPosition = Helper.text_pos(result.tokens, syntax.error_token_index) var spos_main : Helper.StringPosition = Helper.str_pos(document.text, pos_main) var spos_inner : Helper.StringPosition = Helper.str_pos(document.text, pos_inner) var err_text : String = "Syntax error in construction '" + result.tokens[syntax.parse_token_index].text + "'. " err_text += "Unacceptable use '" + result.tokens[syntax.error_token_index].text + "' at:" + str(spos_inner.str_num) + ":" + str(spos_inner.column) err_text += "\n" + syntax.error_description_text printerr(Helper.error_string(document.name, spos_main.str_num, spos_main.column, err_text)) else: analyze_result.version = proto_version analyze_result.imports = import_table analyze_result.syntax = syntax analyze_result.state = true return analyze_result class Semantic: var class_table : Array var field_table : Array var group_table : Array var syntax : Analysis.TranslationResult var tokens : Array var document : Document func _init(analyze_result : AnalyzeResult): class_table = analyze_result.classes field_table = analyze_result.fields group_table = analyze_result.groups syntax = analyze_result.syntax tokens = analyze_result.tokens document = analyze_result.doc enum CHECK_SUBJECT { CLASS_NAME = 0, FIELD_NAME = 1, FIELD_TAG_NUMBER = 2, FIELD_TYPE = 3 } var STRING_FIELD_TYPE = { "int32": Analysis.FIELD_TYPE.INT32, "sint32": Analysis.FIELD_TYPE.SINT32, "uint32": Analysis.FIELD_TYPE.UINT32, "int64": Analysis.FIELD_TYPE.INT64, "sint64": Analysis.FIELD_TYPE.SINT64, "uint64": Analysis.FIELD_TYPE.UINT64, "bool": Analysis.FIELD_TYPE.BOOL, "fixed32": Analysis.FIELD_TYPE.FIXED32, "sfixed32": Analysis.FIELD_TYPE.SFIXED32, "float": Analysis.FIELD_TYPE.FLOAT, "fixed64": Analysis.FIELD_TYPE.FIXED64, "sfixed64": Analysis.FIELD_TYPE.SFIXED64, "double": Analysis.FIELD_TYPE.DOUBLE, "string": Analysis.FIELD_TYPE.STRING, "bytes": Analysis.FIELD_TYPE.BYTES, "map": Analysis.FIELD_TYPE.MAP } class CheckResult: func _init(mci : int, aci : int, ti : int, s : int): main_construction_index = mci associated_construction_index = aci table_index = ti subject = s var main_construction_index: int = -1 var associated_construction_index: int = -1 var table_index: int = -1 var subject : int func check_class_names() -> Array: var result : Array = [] for i in range(class_table.size()): var the_class_name : String = class_table[i].parent_name + "." + class_table[i].name for j in range(i + 1, class_table.size(), 1): var inner_name : String = class_table[j].parent_name + "." + class_table[j].name if inner_name == the_class_name: var check : CheckResult = CheckResult.new(class_table[j].construction_index, class_table[i].construction_index, j, CHECK_SUBJECT.CLASS_NAME) result.append(check) break return result func check_field_names() -> Array: var result : Array = [] for i in range(field_table.size()): var the_class_name : String = class_table[field_table[i].parent_class_id].parent_name + "." + class_table[field_table[i].parent_class_id].name for j in range(i + 1, field_table.size(), 1): var inner_name : String = class_table[field_table[j].parent_class_id].parent_name + "." + class_table[field_table[j].parent_class_id].name if inner_name == the_class_name: if field_table[i].name == field_table[j].name: var check : CheckResult = CheckResult.new(field_table[j].construction_index, field_table[i].construction_index, j, CHECK_SUBJECT.FIELD_NAME) result.append(check) break if field_table[i].tag == field_table[j].tag: var check : CheckResult = CheckResult.new(field_table[j].construction_index, field_table[i].construction_index, j, CHECK_SUBJECT.FIELD_TAG_NUMBER) result.append(check) break return result func find_full_class_name(the_class_name : String) -> int: for i in range(class_table.size()): if the_class_name == class_table[i].parent_name + "." + class_table[i].name: return i return -1 func find_class_name(the_class_name : String) -> int: for i in range(class_table.size()): if the_class_name == class_table[i].name: return i return -1 func get_class_childs(class_index : int) -> Array: var result : Array = [] for i in range(class_table.size()): if class_table[i].parent_index == class_index: result.append(i) return result func find_in_childs(the_class_name : String, child_indexes : Array) -> int: for c in child_indexes: if the_class_name == class_table[c].name: return c return -1 func determine_field_types() -> Array: var result : Array = [] for f in field_table: if STRING_FIELD_TYPE.has(f.type_name): f.field_type = STRING_FIELD_TYPE[f.type_name] else: if f.type_name[0] == ".": f.type_class_id = find_full_class_name(f.type_name) else: # Reset result from previous assignment, that can be incorrect because of merging of imports f.type_class_id = -1 var splited_name : Array = f.type_name.split(".", false) var cur_class_index : int = f.parent_class_id var exit : bool = false while(true): var find : bool = false if cur_class_index == -1: break for n in splited_name: var childs_and_parent : Array = get_class_childs(cur_class_index) var res_index : int = find_in_childs(n, childs_and_parent) if res_index >= 0: find = true cur_class_index = res_index else: if find: exit = true else: cur_class_index = class_table[cur_class_index].parent_index break if exit: break if find: f.type_class_id = cur_class_index break if f.type_class_id == -1: f.type_class_id = find_full_class_name("." + f.type_name) for i in range(field_table.size()): if field_table[i].field_type == Analysis.FIELD_TYPE.UNDEFINED: if field_table[i].type_class_id == -1: result.append(CheckResult.new(field_table[i].construction_index, field_table[i].construction_index, i, CHECK_SUBJECT.FIELD_TYPE)) else: if class_table[field_table[i].type_class_id].type == Analysis.CLASS_TYPE.ENUM: field_table[i].field_type = Analysis.FIELD_TYPE.ENUM elif class_table[field_table[i].type_class_id].type == Analysis.CLASS_TYPE.MESSAGE: field_table[i].field_type = Analysis.FIELD_TYPE.MESSAGE elif class_table[field_table[i].type_class_id].type == Analysis.CLASS_TYPE.MAP: field_table[i].field_type = Analysis.FIELD_TYPE.MAP else: result.append(CheckResult.new(field_table[i].construction_index, field_table[i].construction_index, i, CHECK_SUBJECT.FIELD_TYPE)) return result func check_constructions() -> Array: var cl : Array = check_class_names() var fl : Array = check_field_names() var ft : Array = determine_field_types() return cl + fl + ft func check() -> bool: var check_result : Array = check_constructions() if check_result.size() == 0: return true else: for v in check_result: var main_tok : int = syntax.constructions[v.main_construction_index].begin_token_index var assoc_tok : int = syntax.constructions[v.associated_construction_index].begin_token_index var main_err_pos : Helper.StringPosition = Helper.str_pos(document.text, Helper.text_pos(tokens, main_tok)) var assoc_err_pos : Helper.StringPosition = Helper.str_pos(document.text, Helper.text_pos(tokens, assoc_tok)) var err_text : String if v.subject == CHECK_SUBJECT.CLASS_NAME: var class_type = "Undefined" if class_table[v.table_index].type == Analysis.CLASS_TYPE.ENUM: class_type = "Enum" elif class_table[v.table_index].type == Analysis.CLASS_TYPE.MESSAGE: class_type = "Message" elif class_table[v.table_index].type == Analysis.CLASS_TYPE.MAP: class_type = "Map" err_text = class_type + " name '" + class_table[v.table_index].name + "' is already defined at:" + str(assoc_err_pos.str_num) + ":" + str(assoc_err_pos.column) elif v.subject == CHECK_SUBJECT.FIELD_NAME: err_text = "Field name '" + field_table[v.table_index].name + "' is already defined at:" + str(assoc_err_pos.str_num) + ":" + str(assoc_err_pos.column) elif v.subject == CHECK_SUBJECT.FIELD_TAG_NUMBER: err_text = "Tag number '" + field_table[v.table_index].tag + "' is already defined at:" + str(assoc_err_pos.str_num) + ":" + str(assoc_err_pos.column) elif v.subject == CHECK_SUBJECT.FIELD_TYPE: err_text = "Type '" + field_table[v.table_index].type_name + "' of the '" + field_table[v.table_index].name + "' field undefined" else: err_text = "Undefined error" printerr(Helper.error_string(document.name, main_err_pos.str_num, main_err_pos.column, err_text)) return false class Translator: var class_table : Array var field_table : Array var group_table : Array var proto_version : int func _init(analyzer_result : AnalyzeResult): class_table = analyzer_result.classes field_table = analyzer_result.fields group_table = analyzer_result.groups proto_version = analyzer_result.version func tabulate(text : String, nesting : int) -> String: var tab : String = "" for i in range(nesting): tab += "\t" return tab + text func default_dict_text() -> String: if proto_version == 2: return "DEFAULT_VALUES_2" elif proto_version == 3: return "DEFAULT_VALUES_3" return "TRANSLATION_ERROR" func generate_field_type(field : Analysis.ASTField) -> String: var text : String = "PB_DATA_TYPE." if field.field_type == Analysis.FIELD_TYPE.INT32: return text + "INT32" elif field.field_type == Analysis.FIELD_TYPE.SINT32: return text + "SINT32" elif field.field_type == Analysis.FIELD_TYPE.UINT32: return text + "UINT32" elif field.field_type == Analysis.FIELD_TYPE.INT64: return text + "INT64" elif field.field_type == Analysis.FIELD_TYPE.SINT64: return text + "SINT64" elif field.field_type == Analysis.FIELD_TYPE.UINT64: return text + "UINT64" elif field.field_type == Analysis.FIELD_TYPE.BOOL: return text + "BOOL" elif field.field_type == Analysis.FIELD_TYPE.ENUM: return text + "ENUM" elif field.field_type == Analysis.FIELD_TYPE.FIXED32: return text + "FIXED32" elif field.field_type == Analysis.FIELD_TYPE.SFIXED32: return text + "SFIXED32" elif field.field_type == Analysis.FIELD_TYPE.FLOAT: return text + "FLOAT" elif field.field_type == Analysis.FIELD_TYPE.FIXED64: return text + "FIXED64" elif field.field_type == Analysis.FIELD_TYPE.SFIXED64: return text + "SFIXED64" elif field.field_type == Analysis.FIELD_TYPE.DOUBLE: return text + "DOUBLE" elif field.field_type == Analysis.FIELD_TYPE.STRING: return text + "STRING" elif field.field_type == Analysis.FIELD_TYPE.BYTES: return text + "BYTES" elif field.field_type == Analysis.FIELD_TYPE.MESSAGE: return text + "MESSAGE" elif field.field_type == Analysis.FIELD_TYPE.MAP: return text + "MAP" return text func generate_field_rule(field : Analysis.ASTField) -> String: var text : String = "PB_RULE." if field.qualificator == Analysis.FIELD_QUALIFICATOR.OPTIONAL: return text + "OPTIONAL" elif field.qualificator == Analysis.FIELD_QUALIFICATOR.REQUIRED: return text + "REQUIRED" elif field.qualificator == Analysis.FIELD_QUALIFICATOR.REPEATED: return text + "REPEATED" elif field.qualificator == Analysis.FIELD_QUALIFICATOR.RESERVED: return text + "RESERVED" return text func generate_gdscript_type(field : Analysis.ASTField) -> String: if field.field_type == Analysis.FIELD_TYPE.MESSAGE: var type_name : String = class_table[field.type_class_id].parent_name + "." + class_table[field.type_class_id].name return type_name.substr(1, type_name.length() - 1) return generate_gdscript_simple_type(field) func generate_gdscript_simple_type(field : Analysis.ASTField) -> String: if field.field_type == Analysis.FIELD_TYPE.INT32: return "int" elif field.field_type == Analysis.FIELD_TYPE.SINT32: return "int" elif field.field_type == Analysis.FIELD_TYPE.UINT32: return "int" elif field.field_type == Analysis.FIELD_TYPE.INT64: return "int" elif field.field_type == Analysis.FIELD_TYPE.SINT64: return "int" elif field.field_type == Analysis.FIELD_TYPE.UINT64: return "int" elif field.field_type == Analysis.FIELD_TYPE.BOOL: return "bool" elif field.field_type == Analysis.FIELD_TYPE.ENUM: return "" elif field.field_type == Analysis.FIELD_TYPE.FIXED32: return "int" elif field.field_type == Analysis.FIELD_TYPE.SFIXED32: return "int" elif field.field_type == Analysis.FIELD_TYPE.FLOAT: return "float" elif field.field_type == Analysis.FIELD_TYPE.FIXED64: return "int" elif field.field_type == Analysis.FIELD_TYPE.SFIXED64: return "int" elif field.field_type == Analysis.FIELD_TYPE.DOUBLE: return "float" elif field.field_type == Analysis.FIELD_TYPE.STRING: return "String" elif field.field_type == Analysis.FIELD_TYPE.BYTES: return "PackedByteArray" return "" func generate_field_constructor(field_index : int, nesting : int) -> String: var text : String = "" var f : Analysis.ASTField = field_table[field_index] var field_name : String = "__" + f.name var pbfield_text : String var default_var_name := field_name + "_default" if f.qualificator == Analysis.FIELD_QUALIFICATOR.REPEATED: var type_name := generate_gdscript_type(f) if type_name: text = tabulate("var %s: Array[%s] = []\n" % [default_var_name, type_name], nesting) else: text = tabulate("var %s: Array = []\n" % [default_var_name], nesting) pbfield_text += field_name + " = PBField.new(" pbfield_text += "\"" + f.name + "\", " pbfield_text += generate_field_type(f) + ", " pbfield_text += generate_field_rule(f) + ", " pbfield_text += str(f.tag) + ", " if f.option == Analysis.FIELD_OPTION.PACKED: pbfield_text += "true" elif f.option == Analysis.FIELD_OPTION.NOT_PACKED: pbfield_text += "false" if f.qualificator == Analysis.FIELD_QUALIFICATOR.REPEATED: pbfield_text += ", " + default_var_name else: pbfield_text += ", " + default_dict_text() + "[" + generate_field_type(f) + "]" pbfield_text += ")\n" text += tabulate(pbfield_text, nesting) if f.is_map_field: text += tabulate(field_name + ".is_map_field = true\n", nesting) text += tabulate("service = PBServiceField.new()\n", nesting) text += tabulate("service.field = " + field_name + "\n", nesting) if f.field_type == Analysis.FIELD_TYPE.MESSAGE: if f.qualificator == Analysis.FIELD_QUALIFICATOR.REPEATED: text += tabulate("service.func_ref = Callable(self, \"add_" + f.name + "\")\n", nesting) else: text += tabulate("service.func_ref = Callable(self, \"new_" + f.name + "\")\n", nesting) elif f.field_type == Analysis.FIELD_TYPE.MAP: text += tabulate("service.func_ref = Callable(self, \"add_empty_" + f.name + "\")\n", nesting) text += tabulate("data[" + field_name + ".tag] = service\n", nesting) return text func generate_group_clear(field_index : int, nesting : int) -> String: for g in group_table: var text : String = "" var find : bool = false if g.parent_class_id == field_table[field_index].parent_class_id: for i in g.field_indexes: if field_index == i: find = true text += tabulate("data[" + field_table[i].tag + "].state = PB_SERVICE_STATE.FILLED\n", nesting) else: text += tabulate("__" + field_table[i].name + ".value = " + default_dict_text() + "[" + generate_field_type(field_table[i]) + "]\n", nesting) text += tabulate("data[" + field_table[i].tag + "].state = PB_SERVICE_STATE.UNFILLED\n", nesting) if find: return text return "" func generate_has_oneof(field_index : int, nesting : int) -> String: for g in group_table: var text : String = "" if g.parent_class_id == field_table[field_index].parent_class_id: for i in g.field_indexes: if field_index == i: text += tabulate("func has_" + field_table[i].name + "() -> bool:\n", nesting) nesting += 1 text += tabulate("return data[" + field_table[i].tag + "].state == PB_SERVICE_STATE.FILLED\n", nesting) return text return "" func generate_field(field_index : int, nesting : int) -> String: var text : String = "" var f : Analysis.ASTField = field_table[field_index] var varname : String = "__" + f.name text += tabulate("var " + varname + ": PBField\n", nesting) if f.field_type == Analysis.FIELD_TYPE.MESSAGE: var the_class_name : String = class_table[f.type_class_id].parent_name + "." + class_table[f.type_class_id].name the_class_name = the_class_name.substr(1, the_class_name.length() - 1) if f.qualificator != Analysis.FIELD_QUALIFICATOR.OPTIONAL: text += generate_has_oneof(field_index, nesting) if f.qualificator == Analysis.FIELD_QUALIFICATOR.REPEATED: text += tabulate("func get_" + f.name + "() -> Array[" + the_class_name + "]:\n", nesting) else: if f.qualificator == Analysis.FIELD_QUALIFICATOR.OPTIONAL: text += tabulate("func has_" + f.name + "() -> bool:\n", nesting) nesting += 1 text += tabulate("if " + varname + ".value != null:\n", nesting) nesting += 1 text += tabulate("return true\n", nesting) nesting -= 1 text += tabulate("return false\n", nesting) nesting -= 1 text += tabulate("func get_" + f.name + "() -> " + the_class_name + ":\n", nesting) nesting += 1 text += tabulate("return " + varname + ".value\n", nesting) nesting -= 1 text += tabulate("func clear_" + f.name + "() -> void:\n", nesting) nesting += 1 text += tabulate("data[" + str(f.tag) + "].state = PB_SERVICE_STATE.UNFILLED\n", nesting) if f.qualificator == Analysis.FIELD_QUALIFICATOR.REPEATED: text += tabulate(varname + ".value.clear()\n", nesting) nesting -= 1 text += tabulate("func add_" + f.name + "() -> " + the_class_name + ":\n", nesting) nesting += 1 text += tabulate("var element = " + the_class_name + ".new()\n", nesting) text += tabulate(varname + ".value.append(element)\n", nesting) text += tabulate("return element\n", nesting) else: text += tabulate(varname + ".value = " + default_dict_text() + "[" + generate_field_type(f) + "]\n", nesting) nesting -= 1 text += tabulate("func new_" + f.name + "() -> " + the_class_name + ":\n", nesting) nesting += 1 text += generate_group_clear(field_index, nesting) text += tabulate(varname + ".value = " + the_class_name + ".new()\n", nesting) text += tabulate("return " + varname + ".value\n", nesting) elif f.field_type == Analysis.FIELD_TYPE.MAP: var the_parent_class_name : String = class_table[f.type_class_id].parent_name the_parent_class_name = the_parent_class_name.substr(1, the_parent_class_name.length() - 1) var the_class_name : String = the_parent_class_name + "." + class_table[f.type_class_id].name text += generate_has_oneof(field_index, nesting) text += tabulate("func get_raw_" + f.name + "():\n", nesting) nesting += 1 text += tabulate("return " + varname + ".value\n", nesting) nesting -= 1 text += tabulate("func get_" + f.name + "():\n", nesting) nesting += 1 text += tabulate("return PBPacker.construct_map(" + varname + ".value)\n", nesting) nesting -= 1 text += tabulate("func clear_" + f.name + "():\n", nesting) nesting += 1 text += tabulate("data[" + str(f.tag) + "].state = PB_SERVICE_STATE.UNFILLED\n", nesting) text += tabulate(varname + ".value = " + default_dict_text() + "[" + generate_field_type(f) + "]\n", nesting) nesting -= 1 for i in range(field_table.size()): if field_table[i].parent_class_id == f.type_class_id && field_table[i].name == "value": var gd_type : String = generate_gdscript_simple_type(field_table[i]) var return_type : String = " -> " + the_class_name var value_return_type : String = "" if gd_type != "": value_return_type = return_type elif field_table[i].field_type == Analysis.FIELD_TYPE.MESSAGE: value_return_type = " -> " + the_parent_class_name + "." + field_table[i].type_name text += tabulate("func add_empty_" + f.name + "()" + return_type + ":\n", nesting) nesting += 1 text += generate_group_clear(field_index, nesting) text += tabulate("var element = " + the_class_name + ".new()\n", nesting) text += tabulate(varname + ".value.append(element)\n", nesting) text += tabulate("return element\n", nesting) nesting -= 1 if field_table[i].field_type == Analysis.FIELD_TYPE.MESSAGE: text += tabulate("func add_" + f.name + "(a_key)" + value_return_type + ":\n", nesting) nesting += 1 text += generate_group_clear(field_index, nesting) text += tabulate("var idx = -1\n", nesting) text += tabulate("for i in range(" + varname + ".value.size()):\n", nesting) nesting += 1 text += tabulate("if " + varname + ".value[i].get_key() == a_key:\n", nesting) nesting += 1 text += tabulate("idx = i\n", nesting) text += tabulate("break\n", nesting) nesting -= 2 text += tabulate("var element = " + the_class_name + ".new()\n", nesting) text += tabulate("element.set_key(a_key)\n", nesting) text += tabulate("if idx != -1:\n", nesting) nesting += 1 text += tabulate(varname + ".value[idx] = element\n", nesting) nesting -= 1 text += tabulate("else:\n", nesting) nesting += 1 text += tabulate(varname + ".value.append(element)\n", nesting) nesting -= 1 text += tabulate("return element.new_value()\n", nesting) else: text += tabulate("func add_" + f.name + "(a_key, a_value) -> void:\n", nesting) nesting += 1 text += generate_group_clear(field_index, nesting) text += tabulate("var idx = -1\n", nesting) text += tabulate("for i in range(" + varname + ".value.size()):\n", nesting) nesting += 1 text += tabulate("if " + varname + ".value[i].get_key() == a_key:\n", nesting) nesting += 1 text += tabulate("idx = i\n", nesting) text += tabulate("break\n", nesting) nesting -= 2 text += tabulate("var element = " + the_class_name + ".new()\n", nesting) text += tabulate("element.set_key(a_key)\n", nesting) text += tabulate("element.set_value(a_value)\n", nesting) text += tabulate("if idx != -1:\n", nesting) nesting += 1 text += tabulate(varname + ".value[idx] = element\n", nesting) nesting -= 1 text += tabulate("else:\n", nesting) nesting += 1 text += tabulate(varname + ".value.append(element)\n", nesting) nesting -= 1 break else: var gd_type : String = generate_gdscript_simple_type(f) var return_type : String = "" var argument_type : String = "" if gd_type != "": return_type = " -> " + gd_type argument_type = " : " + gd_type if f.qualificator != Analysis.FIELD_QUALIFICATOR.OPTIONAL: text += generate_has_oneof(field_index, nesting) if f.qualificator == Analysis.FIELD_QUALIFICATOR.REPEATED: var array_type := "[" + gd_type + "]" if gd_type else "" text += tabulate("func get_" + f.name + "() -> Array" + array_type + ":\n", nesting) else: if f.qualificator == Analysis.FIELD_QUALIFICATOR.OPTIONAL: text += tabulate("func has_" + f.name + "() -> bool:\n", nesting) nesting += 1 text += tabulate("if " + varname + ".value != null:\n", nesting) nesting += 1 text += tabulate("return true\n", nesting) nesting -= 1 text += tabulate("return false\n", nesting) nesting -= 1 text += tabulate("func get_" + f.name + "()" + return_type + ":\n", nesting) nesting += 1 text += tabulate("return " + varname + ".value\n", nesting) nesting -= 1 text += tabulate("func clear_" + f.name + "() -> void:\n", nesting) nesting += 1 text += tabulate("data[" + str(f.tag) + "].state = PB_SERVICE_STATE.UNFILLED\n", nesting) if f.qualificator == Analysis.FIELD_QUALIFICATOR.REPEATED: text += tabulate(varname + ".value.clear()\n", nesting) nesting -= 1 text += tabulate("func add_" + f.name + "(value" + argument_type + ") -> void:\n", nesting) nesting += 1 text += tabulate(varname + ".value.append(value)\n", nesting) else: text += tabulate(varname + ".value = " + default_dict_text() + "[" + generate_field_type(f) + "]\n", nesting) nesting -= 1 text += tabulate("func set_" + f.name + "(value" + argument_type + ") -> void:\n", nesting) nesting += 1 text += generate_group_clear(field_index, nesting) text += tabulate(varname + ".value = value\n", nesting) return text func generate_class(class_index : int, nesting : int) -> String: var text : String = "" if class_table[class_index].type == Analysis.CLASS_TYPE.MESSAGE || class_table[class_index].type == Analysis.CLASS_TYPE.MAP: var cls_pref : String = "" cls_pref += tabulate("class " + class_table[class_index].name + ":\n", nesting) nesting += 1 cls_pref += tabulate("func _init():\n", nesting) text += cls_pref nesting += 1 text += tabulate("var service\n", nesting) text += tabulate("\n", nesting) var field_text : String = "" for i in range(field_table.size()): if field_table[i].parent_class_id == class_index: text += generate_field_constructor(i, nesting) text += tabulate("\n", nesting) field_text += generate_field(i, nesting - 1) field_text += tabulate("\n", nesting - 1) nesting -= 1 text += tabulate("var data = {}\n", nesting) text += tabulate("\n", nesting) text += field_text for j in range(class_table.size()): if class_table[j].parent_index == class_index: var cl_text = generate_class(j, nesting) text += cl_text if class_table[j].type == Analysis.CLASS_TYPE.MESSAGE || class_table[j].type == Analysis.CLASS_TYPE.MAP: text += generate_class_services(nesting + 1) text += tabulate("\n", nesting + 1) elif class_table[class_index].type == Analysis.CLASS_TYPE.ENUM: text += tabulate("enum " + class_table[class_index].name + " {\n", nesting) nesting += 1 var expected_prefix = class_table[class_index].name.to_snake_case().to_upper() + "_" var all_have_prefix = true for en in range(class_table[class_index].values.size()): var value_name = class_table[class_index].values[en].name all_have_prefix = all_have_prefix and value_name.begins_with(expected_prefix) and value_name != expected_prefix for en in range(class_table[class_index].values.size()): var value_name = class_table[class_index].values[en].name if all_have_prefix: value_name = value_name.substr(expected_prefix.length()) var enum_val = value_name + " = " + class_table[class_index].values[en].value if en == class_table[class_index].values.size() - 1: text += tabulate(enum_val + "\n", nesting) else: text += tabulate(enum_val + ",\n", nesting) nesting -= 1 text += tabulate("}\n", nesting) text += tabulate("\n", nesting) return text func generate_class_services(nesting : int) -> String: var text : String = "" text += tabulate("func _to_string() -> String:\n", nesting) nesting += 1 text += tabulate("return PBPacker.message_to_string(data)\n", nesting) text += tabulate("\n", nesting) nesting -= 1 text += tabulate("func to_bytes() -> PackedByteArray:\n", nesting) nesting += 1 text += tabulate("return PBPacker.pack_message(data)\n", nesting) text += tabulate("\n", nesting) nesting -= 1 text += tabulate("func from_bytes(bytes : PackedByteArray, offset : int = 0, limit : int = -1) -> int:\n", nesting) nesting += 1 text += tabulate("var cur_limit = bytes.size()\n", nesting) text += tabulate("if limit != -1:\n", nesting) nesting += 1 text += tabulate("cur_limit = limit\n", nesting) nesting -= 1 text += tabulate("var result = PBPacker.unpack_message(data, bytes, offset, cur_limit)\n", nesting) text += tabulate("if result == cur_limit:\n", nesting) nesting += 1 text += tabulate("if PBPacker.check_required(data):\n", nesting) nesting += 1 text += tabulate("if limit == -1:\n", nesting) nesting += 1 text += tabulate("return PB_ERR.NO_ERRORS\n", nesting) nesting -= 2 text += tabulate("else:\n", nesting) nesting += 1 text += tabulate("return PB_ERR.REQUIRED_FIELDS\n", nesting) nesting -= 2 text += tabulate("elif limit == -1 && result > 0:\n", nesting) nesting += 1 text += tabulate("return PB_ERR.PARSE_INCOMPLETE\n", nesting) nesting -= 1 text += tabulate("return result\n", nesting) return text func translate(file_name : String, core_file_name : String) -> bool: var file : FileAccess = FileAccess.open(file_name, FileAccess.WRITE) if file == null: printerr("File: '", file_name, "' save error.") return false if !FileAccess.file_exists(core_file_name): printerr("File: '", core_file_name, "' not found.") return false var core_file : FileAccess = FileAccess.open(core_file_name, FileAccess.READ) if core_file == null: printerr("File: '", core_file_name, "' read error.") return false var core_text : String = core_file.get_as_text() core_file.close() var text : String = "" var nesting : int = 0 core_text = core_text.replace(PROTO_VERSION_DEFAULT, PROTO_VERSION_CONST + str(proto_version)) text += core_text + "\n\n\n" text += "############### USER DATA BEGIN ################\n" var cls_user : String = "" for i in range(class_table.size()): if class_table[i].parent_index == -1: var cls_text = generate_class(i, nesting) cls_user += cls_text if class_table[i].type == Analysis.CLASS_TYPE.MESSAGE: nesting += 1 cls_user += generate_class_services(nesting) cls_user += tabulate("\n", nesting) nesting -= 1 text += "\n\n" text += cls_user text += "################ USER DATA END #################\n" file.store_string(text) file.close() if !FileAccess.file_exists(file_name): printerr("File: '", file_name, "' save error.") return false return true class ImportFile: func _init(sha : String, a_path : String, a_parent : int): sha256 = sha path = a_path parent_index = a_parent var sha256 : String var path : String var parent_index : int func parse_all(analyzes : Dictionary, imports : Array, path : String, full_name : String, parent_index : int) -> bool: if !FileAccess.file_exists(full_name): printerr(full_name, ": not found.") return false var file : FileAccess = FileAccess.open(full_name, FileAccess.READ) if file == null: printerr(full_name, ": read error.") return false var doc : Document = Document.new(full_name, file.get_as_text()) var sha : String = file.get_sha256(full_name) file.close() if !analyzes.has(sha): print(full_name, ": parsing.") var analysis : Analysis = Analysis.new(path, doc) var an_result : AnalyzeResult = analysis.analyze() if an_result.state: analyzes[sha] = an_result var parent : int = imports.size() imports.append(ImportFile.new(sha, doc.name, parent_index)) for im in an_result.imports: if !parse_all(analyzes, imports, path, im.path, parent): return false else: printerr(doc.name + ": parsing error.") return false else: print(full_name, ": retrieving data from cache.") imports.append(ImportFile.new(sha, doc.name, parent_index)) return true func union_analyses(a1 : AnalyzeResult, a2 : AnalyzeResult, only_classes : bool = true) -> void: var class_offset : int = a1.classes.size() var field_offset = a1.fields.size() for cl in a2.classes: var cur_class : Analysis.ASTClass = cl.copy() if cur_class.parent_index != -1: cur_class.parent_index += class_offset a1.classes.append(cur_class) if only_classes: return for fl in a2.fields: var cur_field : Analysis.ASTField = fl.copy() cur_field.parent_class_id += class_offset cur_field.type_class_id = -1 a1.fields.append(cur_field) for gr in a2.groups: var cur_group : Analysis.ASTFieldGroup = gr.copy() cur_group.parent_class_id += class_offset var indexes : Array = [] for i in cur_group.field_indexes: indexes.append(i + field_offset) cur_group.field_indexes = indexes a1.groups.append(cur_group) func union_imports(analyzes : Dictionary, key : String, result : AnalyzeResult, keys : Array, nesting : int, use_public : bool = true, only_classes : bool = true) -> void: nesting += 1 for im in analyzes[key].imports: var find : bool = false for k in keys: if im.sha256 == k: find = true break if find: continue if (!use_public) || (use_public && ((im.public && nesting > 1) || nesting < 2)): keys.append(im.sha256) union_analyses(result, analyzes[im.sha256], only_classes) union_imports(analyzes, im.sha256, result, keys, nesting, use_public, only_classes) func semantic_all(analyzes : Dictionary, imports : Array)-> bool: for k in analyzes.keys(): print(analyzes[k].doc.name, ": analysis.") var keys : Array = [] var analyze : AnalyzeResult = analyzes[k].soft_copy() keys.append(k) analyze.classes = [] for cl in analyzes[k].classes: analyze.classes.append(cl.copy()) union_imports(analyzes, k, analyze, keys, 0) var semantic : Semantic = Semantic.new(analyze) if !semantic.check(): printerr(analyzes[k].doc.name, ": analysis error.") return false return true func translate_all(analyzes : Dictionary, file_name : String, core_file_name : String) -> bool: var first_key : String = analyzes.keys()[0] var analyze : AnalyzeResult = analyzes[first_key] var keys : Array = [] keys.append(first_key) union_imports(analyzes, first_key, analyze, keys, 0, false, false) print("Performing full semantic analysis.") var semantic : Semantic = Semantic.new(analyze) if !semantic.check(): return false print("Performing translation.") var translator : Translator = Translator.new(analyze) if !translator.translate(file_name, core_file_name): return false var first : bool = true return true func work(path : String, in_file : String, out_file : String, core_file : String) -> bool: var in_full_name : String = path + in_file var imports : Array = [] var analyzes : Dictionary = {} print("Compiling source: '", in_full_name, "', output: '", out_file, "'.") print("\n1. Parsing:") if parse_all(analyzes, imports, path, in_full_name, -1): print("* Parsing completed successfully. *") else: return false print("\n2. Perfoming semantic analysis:") if semantic_all(analyzes, imports): print("* Semantic analysis completed successfully. *") else: return false print("\n3. Output file creating:") if translate_all(analyzes, out_file, core_file): print("* Output file was created successfully. *") else: return false return true func _ready(): pass