class Demo::RubyTokenizer
Constants
- RESERVED_WORDS
- RESERVED_WORDS_PATTERN
Public Instance Methods
tokenize(str, index = 0) { |*args| ... }
click to toggle source
# File gtk3/sample/gtk-demo/main.rb, line 485 def tokenize(str, index = 0) until str.empty? tag = nil case str when /".+?"/, /'.+?'/ tag = :string when /#.*$/ tag = :comment when RESERVED_WORDS_PATTERN tag = :reserved when /[A-Z][A-Za-z0-9_]+/ tag = :const end if tag tokenize($LAST_MATCH_INFO.pre_match, index) do |*args| yield(*args) end yield(tag, index + $LAST_MATCH_INFO.begin(0), index + $LAST_MATCH_INFO.end(0)) index += (str.length - $LAST_MATCH_INFO.post_match.length) str = $LAST_MATCH_INFO.post_match else index += str.length str = "" end end end