Class | CodeRay::Scanners::JavaScript |
In: |
lib/coderay/scanners/java_script.rb
lib/coderay/scanners/java_script-0.9.6.rb |
Parent: | Scanner |
KEYWORDS | = | %w[ break case catch continue default delete do else finally for function if in instanceof new return switch throw try typeof var void while with ] | The actual JavaScript keywords. | |
PREDEFINED_CONSTANTS | = | %w[ false null true undefined ] | ||
MAGIC_VARIABLES | = | %w[ this arguments ] | ||
KEYWORDS_EXPECTING_VALUE | = | WordList.new.add %w[ case delete in instanceof new return throw typeof with ] | ||
RESERVED_WORDS | = | %w[ abstract boolean byte char class debugger double enum export extends final float goto implements import int interface long native package private protected public short static super synchronized throws transient volatile ] | Reserved for future use. | |
IDENT_KIND | = | WordList.new(:ident). add(RESERVED_WORDS, :reserved). add(PREDEFINED_CONSTANTS, :pre_constant). add(MAGIC_VARIABLES, :local_variable). add(KEYWORDS, :keyword) | ||
ESCAPE | = | / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x | ||
UNICODE_ESCAPE | = | / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x | ||
REGEXP_ESCAPE | = | / [bBdDsSwW] /x | ||
STRING_CONTENT_PATTERN | = | { "'" => /[^\\']+/, '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } | ||
KEY_CHECK_PATTERN | = | { "'" => / (?> [^\\']* (?: \\. [^\\']* )* ) ' \s* : /mx, '"' => / (?> [^\\"]* (?: \\. [^\\"]* )* ) " \s* : /mx, } | ||
KEYWORDS | = | %w[ break case catch continue default delete do else finally for function if in instanceof new return switch throw try typeof var void while with ] | The actual JavaScript keywords. | |
PREDEFINED_CONSTANTS | = | %w[ false null true undefined ] | ||
MAGIC_VARIABLES | = | %w[ this arguments ] | ||
KEYWORDS_EXPECTING_VALUE | = | WordList.new.add %w[ case delete in instanceof new return throw typeof with ] | ||
RESERVED_WORDS | = | %w[ abstract boolean byte char class debugger double enum export extends final float goto implements import int interface long native package private protected public short static super synchronized throws transient volatile ] | Reserved for future use. | |
IDENT_KIND | = | WordList.new(:ident). add(RESERVED_WORDS, :reserved). add(PREDEFINED_CONSTANTS, :pre_constant). add(MAGIC_VARIABLES, :local_variable). add(KEYWORDS, :keyword) | ||
ESCAPE | = | / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x | ||
UNICODE_ESCAPE | = | / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x | ||
REGEXP_ESCAPE | = | / [bBdDsSwW] /x | ||
STRING_CONTENT_PATTERN | = | { "'" => /[^\\']+/, '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } | ||
KEY_CHECK_PATTERN | = | { "'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x, '"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x, } |
# File lib/coderay/scanners/java_script.rb, line 54 54: def scan_tokens tokens, options 55: 56: state = :initial 57: string_delimiter = nil 58: value_expected = true 59: key_expected = false 60: function_expected = false 61: 62: until eos? 63: 64: kind = nil 65: match = nil 66: 67: case state 68: 69: when :initial 70: 71: if match = scan(/ \s+ | \\\n /x) 72: value_expected = true if !value_expected && match.index(?\n) 73: tokens << [match, :space] 74: next 75: 76: elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) 77: value_expected = true 78: kind = :comment 79: 80: elsif check(/\.?\d/) 81: key_expected = value_expected = false 82: if scan(/0[xX][0-9A-Fa-f]+/) 83: kind = :hex 84: elsif scan(/(?>0[0-7]+)(?![89.eEfF])/) 85: kind = :oct 86: elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) 87: kind = :float 88: elsif scan(/\d+/) 89: kind = :integer 90: end 91: 92: elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim) 93: # FIXME: scan over nested tags 94: xml_scanner.tokenize match 95: value_expected = false 96: next 97: 98: elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x) 99: value_expected = true 100: last_operator = match[-1] 101: key_expected = (last_operator == ?{) || (last_operator == ?,) 102: function_expected = false 103: kind = :operator 104: 105: elsif scan(/ [)\]}]+ /x) 106: function_expected = key_expected = value_expected = false 107: kind = :operator 108: 109: elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x) 110: kind = IDENT_KIND[match] 111: value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match] 112: # TODO: labels 113: if kind == :ident 114: if match.index(?$) # $ allowed inside an identifier 115: kind = :predefined 116: elsif function_expected 117: kind = :function 118: elsif check(/\s*[=:]\s*function\b/) 119: kind = :function 120: elsif key_expected && check(/\s*:/) 121: kind = :key 122: end 123: end 124: function_expected = (kind == :keyword) && (match == 'function') 125: key_expected = false 126: 127: elsif match = scan(/["']/) 128: if key_expected && check(KEY_CHECK_PATTERN[match]) 129: state = :key 130: else 131: state = :string 132: end 133: tokens << [:open, state] 134: string_delimiter = match 135: kind = :delimiter 136: 137: elsif value_expected && (match = scan(/\/(?=\S)/)) 138: tokens << [:open, :regexp] 139: state = :regexp 140: string_delimiter = '/' 141: kind = :delimiter 142: 143: elsif scan(/ \/ /x) 144: value_expected = true 145: key_expected = false 146: kind = :operator 147: 148: else 149: getch 150: kind = :error 151: 152: end 153: 154: when :string, :regexp, :key 155: if scan(STRING_CONTENT_PATTERN[string_delimiter]) 156: kind = :content 157: elsif match = scan(/["'\/]/) 158: tokens << [match, :delimiter] 159: if state == :regexp 160: modifiers = scan(/[gim]+/) 161: tokens << [modifiers, :modifier] if modifiers && !modifiers.empty? 162: end 163: tokens << [:close, state] 164: string_delimiter = nil 165: key_expected = value_expected = false 166: state = :initial 167: next 168: elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)) 169: if string_delimiter == "'" && !(match == "\\\\" || match == "\\'") 170: kind = :content 171: else 172: kind = :char 173: end 174: elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox) 175: kind = :char 176: elsif scan(/\\./m) 177: kind = :content 178: elsif scan(/ \\ | $ /x) 179: tokens << [:close, state] 180: kind = :error 181: key_expected = value_expected = false 182: state = :initial 183: else 184: raise_inspect "else case \" reached; %p not handled." % peek(1), tokens 185: end 186: 187: else 188: raise_inspect 'Unknown state', tokens 189: 190: end 191: 192: match ||= matched 193: if $CODERAY_DEBUG and not kind 194: raise_inspect 'Error token %p in line %d' % 195: [[match, kind], line], tokens 196: end 197: raise_inspect 'Empty token', tokens unless match 198: 199: tokens << [match, kind] 200: 201: end 202: 203: if [:string, :regexp].include? state 204: tokens << [:close, state] 205: end 206: 207: tokens 208: end
# File lib/coderay/scanners/java_script-0.9.6.rb, line 54 54: def scan_tokens tokens, options 55: 56: state = :initial 57: string_delimiter = nil 58: value_expected = true 59: key_expected = false 60: function_expected = false 61: 62: until eos? 63: 64: kind = nil 65: match = nil 66: 67: case state 68: 69: when :initial 70: 71: if match = scan(/ \s+ | \\\n /x) 72: value_expected = true if !value_expected && match.index(?\n) 73: tokens << [match, :space] 74: next 75: 76: elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) 77: value_expected = true 78: kind = :comment 79: 80: elsif check(/\.?\d/) 81: key_expected = value_expected = false 82: if scan(/0[xX][0-9A-Fa-f]+/) 83: kind = :hex 84: elsif scan(/(?>0[0-7]+)(?![89.eEfF])/) 85: kind = :oct 86: elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) 87: kind = :float 88: elsif scan(/\d+/) 89: kind = :integer 90: end 91: 92: elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim) 93: # FIXME: scan over nested tags 94: xml_scanner.tokenize match 95: value_expected = false 96: next 97: 98: elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x) 99: value_expected = true 100: last_operator = match[-1] 101: key_expected = (last_operator == ?{) || (last_operator == ?,) 102: function_expected = false 103: kind = :operator 104: 105: elsif scan(/ [)\]}]+ /x) 106: function_expected = key_expected = value_expected = false 107: kind = :operator 108: 109: elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x) 110: kind = IDENT_KIND[match] 111: value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match] 112: # TODO: labels 113: if kind == :ident 114: if match.index(?$) # $ allowed inside an identifier 115: kind = :predefined 116: elsif function_expected 117: kind = :function 118: elsif check(/\s*[=:]\s*function\b/) 119: kind = :function 120: elsif key_expected && check(/\s*:/) 121: kind = :key 122: end 123: end 124: function_expected = (kind == :keyword) && (match == 'function') 125: key_expected = false 126: 127: elsif match = scan(/["']/) 128: if key_expected && check(KEY_CHECK_PATTERN[match]) 129: state = :key 130: else 131: state = :string 132: end 133: tokens << [:open, state] 134: string_delimiter = match 135: kind = :delimiter 136: 137: elsif value_expected && (match = scan(/\/(?=\S)/)) 138: tokens << [:open, :regexp] 139: state = :regexp 140: string_delimiter = '/' 141: kind = :delimiter 142: 143: elsif scan(/ \/ /x) 144: value_expected = true 145: key_expected = false 146: kind = :operator 147: 148: else 149: getch 150: kind = :error 151: 152: end 153: 154: when :string, :regexp, :key 155: if scan(STRING_CONTENT_PATTERN[string_delimiter]) 156: kind = :content 157: elsif match = scan(/["'\/]/) 158: tokens << [match, :delimiter] 159: if state == :regexp 160: modifiers = scan(/[gim]+/) 161: tokens << [modifiers, :modifier] if modifiers && !modifiers.empty? 162: end 163: tokens << [:close, state] 164: string_delimiter = nil 165: key_expected = value_expected = false 166: state = :initial 167: next 168: elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)) 169: if string_delimiter == "'" && !(match == "\\\\" || match == "\\'") 170: kind = :content 171: else 172: kind = :char 173: end 174: elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox) 175: kind = :char 176: elsif scan(/\\./m) 177: kind = :content 178: elsif scan(/ \\ | $ /x) 179: tokens << [:close, state] 180: kind = :error 181: key_expected = value_expected = false 182: state = :initial 183: else 184: raise_inspect "else case \" reached; %p not handled." % peek(1), tokens 185: end 186: 187: else 188: raise_inspect 'Unknown state', tokens 189: 190: end 191: 192: match ||= matched 193: if $CODERAY_DEBUG and not kind 194: raise_inspect 'Error token %p in line %d' % 195: [[match, kind], line], tokens 196: end 197: raise_inspect 'Empty token', tokens unless match 198: 199: tokens << [match, kind] 200: 201: end 202: 203: if [:string, :regexp].include? state 204: tokens << [:close, state] 205: end 206: 207: tokens 208: end
# File lib/coderay/scanners/java_script-0.9.6.rb, line 212 212: def reset_instance 213: super 214: @xml_scanner.reset if defined? @xml_scanner 215: end
# File lib/coderay/scanners/java_script.rb, line 212 212: def reset_instance 213: super 214: @xml_scanner.reset if defined? @xml_scanner 215: end
# File lib/coderay/scanners/java_script.rb, line 217 217: def xml_scanner 218: @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false 219: end