1 /** 2 * Compiler implementation of the 3 * $(LINK2 http://www.dlang.org, D programming language). 4 * 5 * Copyright: Copyright (c) 1999-2016 by Digital Mars, All Rights Reserved 6 * Authors: $(LINK2 http://www.digitalmars.com, Walter Bright) 7 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) 8 * Source: $(DMDSRC _tokens.d) 9 */ 10 11 module ddmd.tokens; 12 13 import core.stdc.ctype; 14 import core.stdc.stdio; 15 import core.stdc..string; 16 import ddmd.globals; 17 import ddmd.identifier; 18 import ddmd.root.ctfloat; 19 import ddmd.root.outbuffer; 20 import ddmd.root.rmem; 21 import ddmd.utf; 22 23 enum TOK : int 24 { 25 TOKreserved, 26 27 // Other 28 TOKlparen, 29 TOKrparen, 30 TOKlbracket, 31 TOKrbracket, 32 TOKlcurly, 33 TOKrcurly, 34 TOKcolon, 35 TOKneg, 36 TOKsemicolon, 37 TOKdotdotdot, 38 TOKeof, 39 TOKcast, 40 TOKnull, 41 TOKassert, 42 TOKtrue, 43 TOKfalse, 44 TOKarray, 45 TOKcall, 46 TOKaddress, 47 TOKtype, 48 TOKthrow, 49 TOKnew, 50 TOKdelete, 51 TOKstar, 52 TOKsymoff, 53 TOKvar, 54 TOKdotvar, 55 TOKdotid, 56 TOKdotti, 57 TOKdottype, 58 TOKslice, 59 TOKarraylength, 60 TOKversion, 61 TOKmodule, 62 TOKdollar, 63 TOKtemplate, 64 TOKdottd, 65 TOKdeclaration, 66 TOKtypeof, 67 TOKpragma, 68 TOKdsymbol, 69 TOKtypeid, 70 TOKuadd, 71 TOKremove, 72 TOKnewanonclass, 73 TOKcomment, 74 TOKarrayliteral, 75 TOKassocarrayliteral, 76 TOKstructliteral, 77 TOKclassreference, 78 TOKthrownexception, 79 TOKdelegateptr, 80 TOKdelegatefuncptr, 81 82 // 54 83 // Operators 84 TOKlt, 85 TOKgt, 86 TOKle, 87 TOKge, 88 TOKequal, 89 TOKnotequal, 90 TOKidentity, 91 TOKnotidentity, 92 TOKindex, 93 TOKis, 94 95 // 64 96 // NCEG floating point compares 97 // !<>= <> <>= !> !>= !< !<= !<> 98 TOKunord, 99 TOKlg, 100 TOKleg, 101 TOKule, 102 TOKul, 103 TOKuge, 104 TOKug, 105 TOKue, 106 107 // 72 108 TOKshl, 109 TOKshr, 110 TOKshlass, 111 TOKshrass, 112 TOKushr, 113 TOKushrass, 114 TOKcat, 115 TOKcatass, // ~ ~= 116 TOKadd, 117 TOKmin, 118 TOKaddass, 119 TOKminass, 120 TOKmul, 121 TOKdiv, 122 TOKmod, 123 TOKmulass, 124 TOKdivass, 125 TOKmodass, 126 TOKand, 127 TOKor, 128 TOKxor, 129 TOKandass, 130 TOKorass, 131 TOKxorass, 132 TOKassign, 133 TOKnot, 134 TOKtilde, 135 TOKplusplus, 136 TOKminusminus, 137 TOKconstruct, 138 TOKblit, 139 TOKdot, 140 TOKarrow, 141 TOKcomma, 142 TOKquestion, 143 TOKandand, 144 TOKoror, 145 TOKpreplusplus, 146 TOKpreminusminus, 147 148 // 111 149 // Numeric literals 150 TOKint32v, 151 TOKuns32v, 152 TOKint64v, 153 TOKuns64v, 154 TOKint128v, 155 TOKuns128v, 156 TOKfloat32v, 157 TOKfloat64v, 158 TOKfloat80v, 159 TOKimaginary32v, 160 TOKimaginary64v, 161 TOKimaginary80v, 162 163 // 123 164 // Char constants 165 TOKcharv, 166 TOKwcharv, 167 TOKdcharv, 168 169 // 126 170 // Leaf operators 171 TOKidentifier, 172 TOKstring, 173 TOKxstring, 174 TOKthis, 175 TOKsuper, 176 TOKhalt, 177 TOKtuple, 178 TOKerror, 179 180 // 134 181 // Basic types 182 TOKvoid, 183 TOKint8, 184 TOKuns8, 185 TOKint16, 186 TOKuns16, 187 TOKint32, 188 TOKuns32, 189 TOKint64, 190 TOKuns64, 191 TOKint128, 192 TOKuns128, 193 TOKfloat32, 194 TOKfloat64, 195 TOKfloat80, 196 TOKimaginary32, 197 TOKimaginary64, 198 TOKimaginary80, 199 TOKcomplex32, 200 TOKcomplex64, 201 TOKcomplex80, 202 TOKchar, 203 TOKwchar, 204 TOKdchar, 205 TOKbool, 206 207 // 158 208 // Aggregates 209 TOKstruct, 210 TOKclass, 211 TOKinterface, 212 TOKunion, 213 TOKenum, 214 TOKimport, 215 TOKalias, 216 TOKoverride, 217 TOKdelegate, 218 TOKfunction, 219 TOKmixin, 220 TOKalign, 221 TOKextern, 222 TOKprivate, 223 TOKprotected, 224 TOKpublic, 225 TOKexport, 226 TOKstatic, 227 TOKfinal, 228 TOKconst, 229 TOKabstract, 230 TOKdebug, 231 TOKdeprecated, 232 TOKin, 233 TOKout, 234 TOKinout, 235 TOKlazy, 236 TOKauto, 237 TOKpackage, 238 TOKmanifest, 239 TOKimmutable, 240 241 // 189 242 // Statements 243 TOKif, 244 TOKelse, 245 TOKwhile, 246 TOKfor, 247 TOKdo, 248 TOKswitch, 249 TOKcase, 250 TOKdefault, 251 TOKbreak, 252 TOKcontinue, 253 TOKwith, 254 TOKsynchronized, 255 TOKreturn, 256 TOKgoto, 257 TOKtry, 258 TOKcatch, 259 TOKfinally, 260 TOKasm, 261 TOKforeach, 262 TOKforeach_reverse, 263 TOKscope, 264 TOKon_scope_exit, 265 TOKon_scope_failure, 266 TOKon_scope_success, 267 268 // 213 269 // Contracts 270 TOKbody, 271 TOKinvariant, 272 273 // Testing 274 TOKunittest, 275 276 // Added after 1.0 277 TOKargTypes, 278 TOKref, 279 TOKmacro, 280 281 // 219 282 TOKparameters, 283 TOKtraits, 284 TOKoverloadset, 285 TOKpure, 286 TOKnothrow, 287 TOKgshared, 288 TOKline, 289 TOKfile, 290 TOKfilefullpath, 291 TOKmodulestring, 292 TOKfuncstring, 293 TOKprettyfunc, 294 TOKshared, 295 TOKat, 296 TOKpow, 297 TOKpowass, 298 TOKgoesto, 299 TOKvector, 300 TOKpound, 301 302 // 237 303 TOKinterval, 304 TOKvoidexp, 305 TOKcantexp, 306 307 TOKMAX, 308 } 309 310 alias TOKreserved = TOK.TOKreserved; 311 alias TOKlparen = TOK.TOKlparen; 312 alias TOKrparen = TOK.TOKrparen; 313 alias TOKlbracket = TOK.TOKlbracket; 314 alias TOKrbracket = TOK.TOKrbracket; 315 alias TOKlcurly = TOK.TOKlcurly; 316 alias TOKrcurly = TOK.TOKrcurly; 317 alias TOKcolon = TOK.TOKcolon; 318 alias TOKneg = TOK.TOKneg; 319 alias TOKsemicolon = TOK.TOKsemicolon; 320 alias TOKdotdotdot = TOK.TOKdotdotdot; 321 alias TOKeof = TOK.TOKeof; 322 alias TOKcast = TOK.TOKcast; 323 alias TOKnull = TOK.TOKnull; 324 alias TOKassert = TOK.TOKassert; 325 alias TOKtrue = TOK.TOKtrue; 326 alias TOKfalse = TOK.TOKfalse; 327 alias TOKarray = TOK.TOKarray; 328 alias TOKcall = TOK.TOKcall; 329 alias TOKaddress = TOK.TOKaddress; 330 alias TOKtype = TOK.TOKtype; 331 alias TOKthrow = TOK.TOKthrow; 332 alias TOKnew = TOK.TOKnew; 333 alias TOKdelete = TOK.TOKdelete; 334 alias TOKstar = TOK.TOKstar; 335 alias TOKsymoff = TOK.TOKsymoff; 336 alias TOKvar = TOK.TOKvar; 337 alias TOKdotvar = TOK.TOKdotvar; 338 alias TOKdotid = TOK.TOKdotid; 339 alias TOKdotti = TOK.TOKdotti; 340 alias TOKdottype = TOK.TOKdottype; 341 alias TOKslice = TOK.TOKslice; 342 alias TOKarraylength = TOK.TOKarraylength; 343 alias TOKversion = TOK.TOKversion; 344 alias TOKmodule = TOK.TOKmodule; 345 alias TOKdollar = TOK.TOKdollar; 346 alias TOKtemplate = TOK.TOKtemplate; 347 alias TOKdottd = TOK.TOKdottd; 348 alias TOKdeclaration = TOK.TOKdeclaration; 349 alias TOKtypeof = TOK.TOKtypeof; 350 alias TOKpragma = TOK.TOKpragma; 351 alias TOKdsymbol = TOK.TOKdsymbol; 352 alias TOKtypeid = TOK.TOKtypeid; 353 alias TOKuadd = TOK.TOKuadd; 354 alias TOKremove = TOK.TOKremove; 355 alias TOKnewanonclass = TOK.TOKnewanonclass; 356 alias TOKcomment = TOK.TOKcomment; 357 alias TOKarrayliteral = TOK.TOKarrayliteral; 358 alias TOKassocarrayliteral = TOK.TOKassocarrayliteral; 359 alias TOKstructliteral = TOK.TOKstructliteral; 360 alias TOKclassreference = TOK.TOKclassreference; 361 alias TOKthrownexception = TOK.TOKthrownexception; 362 alias TOKdelegateptr = TOK.TOKdelegateptr; 363 alias TOKdelegatefuncptr = TOK.TOKdelegatefuncptr; 364 alias TOKlt = TOK.TOKlt; 365 alias TOKgt = TOK.TOKgt; 366 alias TOKle = TOK.TOKle; 367 alias TOKge = TOK.TOKge; 368 alias TOKequal = TOK.TOKequal; 369 alias TOKnotequal = TOK.TOKnotequal; 370 alias TOKidentity = TOK.TOKidentity; 371 alias TOKnotidentity = TOK.TOKnotidentity; 372 alias TOKindex = TOK.TOKindex; 373 alias TOKis = TOK.TOKis; 374 alias TOKunord = TOK.TOKunord; 375 alias TOKlg = TOK.TOKlg; 376 alias TOKleg = TOK.TOKleg; 377 alias TOKule = TOK.TOKule; 378 alias TOKul = TOK.TOKul; 379 alias TOKuge = TOK.TOKuge; 380 alias TOKug = TOK.TOKug; 381 alias TOKue = TOK.TOKue; 382 alias TOKshl = TOK.TOKshl; 383 alias TOKshr = TOK.TOKshr; 384 alias TOKshlass = TOK.TOKshlass; 385 alias TOKshrass = TOK.TOKshrass; 386 alias TOKushr = TOK.TOKushr; 387 alias TOKushrass = TOK.TOKushrass; 388 alias TOKcat = TOK.TOKcat; 389 alias TOKcatass = TOK.TOKcatass; 390 alias TOKadd = TOK.TOKadd; 391 alias TOKmin = TOK.TOKmin; 392 alias TOKaddass = TOK.TOKaddass; 393 alias TOKminass = TOK.TOKminass; 394 alias TOKmul = TOK.TOKmul; 395 alias TOKdiv = TOK.TOKdiv; 396 alias TOKmod = TOK.TOKmod; 397 alias TOKmulass = TOK.TOKmulass; 398 alias TOKdivass = TOK.TOKdivass; 399 alias TOKmodass = TOK.TOKmodass; 400 alias TOKand = TOK.TOKand; 401 alias TOKor = TOK.TOKor; 402 alias TOKxor = TOK.TOKxor; 403 alias TOKandass = TOK.TOKandass; 404 alias TOKorass = TOK.TOKorass; 405 alias TOKxorass = TOK.TOKxorass; 406 alias TOKassign = TOK.TOKassign; 407 alias TOKnot = TOK.TOKnot; 408 alias TOKtilde = TOK.TOKtilde; 409 alias TOKplusplus = TOK.TOKplusplus; 410 alias TOKminusminus = TOK.TOKminusminus; 411 alias TOKconstruct = TOK.TOKconstruct; 412 alias TOKblit = TOK.TOKblit; 413 alias TOKdot = TOK.TOKdot; 414 alias TOKarrow = TOK.TOKarrow; 415 alias TOKcomma = TOK.TOKcomma; 416 alias TOKquestion = TOK.TOKquestion; 417 alias TOKandand = TOK.TOKandand; 418 alias TOKoror = TOK.TOKoror; 419 alias TOKpreplusplus = TOK.TOKpreplusplus; 420 alias TOKpreminusminus = TOK.TOKpreminusminus; 421 alias TOKint32v = TOK.TOKint32v; 422 alias TOKuns32v = TOK.TOKuns32v; 423 alias TOKint64v = TOK.TOKint64v; 424 alias TOKuns64v = TOK.TOKuns64v; 425 alias TOKint128v = TOK.TOKint128v; 426 alias TOKuns128v = TOK.TOKuns128v; 427 alias TOKfloat32v = TOK.TOKfloat32v; 428 alias TOKfloat64v = TOK.TOKfloat64v; 429 alias TOKfloat80v = TOK.TOKfloat80v; 430 alias TOKimaginary32v = TOK.TOKimaginary32v; 431 alias TOKimaginary64v = TOK.TOKimaginary64v; 432 alias TOKimaginary80v = TOK.TOKimaginary80v; 433 alias TOKcharv = TOK.TOKcharv; 434 alias TOKwcharv = TOK.TOKwcharv; 435 alias TOKdcharv = TOK.TOKdcharv; 436 alias TOKidentifier = TOK.TOKidentifier; 437 alias TOKstring = TOK.TOKstring; 438 alias TOKxstring = TOK.TOKxstring; 439 alias TOKthis = TOK.TOKthis; 440 alias TOKsuper = TOK.TOKsuper; 441 alias TOKhalt = TOK.TOKhalt; 442 alias TOKtuple = TOK.TOKtuple; 443 alias TOKerror = TOK.TOKerror; 444 alias TOKvoid = TOK.TOKvoid; 445 alias TOKint8 = TOK.TOKint8; 446 alias TOKuns8 = TOK.TOKuns8; 447 alias TOKint16 = TOK.TOKint16; 448 alias TOKuns16 = TOK.TOKuns16; 449 alias TOKint32 = TOK.TOKint32; 450 alias TOKuns32 = TOK.TOKuns32; 451 alias TOKint64 = TOK.TOKint64; 452 alias TOKuns64 = TOK.TOKuns64; 453 alias TOKint128 = TOK.TOKint128; 454 alias TOKuns128 = TOK.TOKuns128; 455 alias TOKfloat32 = TOK.TOKfloat32; 456 alias TOKfloat64 = TOK.TOKfloat64; 457 alias TOKfloat80 = TOK.TOKfloat80; 458 alias TOKimaginary32 = TOK.TOKimaginary32; 459 alias TOKimaginary64 = TOK.TOKimaginary64; 460 alias TOKimaginary80 = TOK.TOKimaginary80; 461 alias TOKcomplex32 = TOK.TOKcomplex32; 462 alias TOKcomplex64 = TOK.TOKcomplex64; 463 alias TOKcomplex80 = TOK.TOKcomplex80; 464 alias TOKchar = TOK.TOKchar; 465 alias TOKwchar = TOK.TOKwchar; 466 alias TOKdchar = TOK.TOKdchar; 467 alias TOKbool = TOK.TOKbool; 468 alias TOKstruct = TOK.TOKstruct; 469 alias TOKclass = TOK.TOKclass; 470 alias TOKinterface = TOK.TOKinterface; 471 alias TOKunion = TOK.TOKunion; 472 alias TOKenum = TOK.TOKenum; 473 alias TOKimport = TOK.TOKimport; 474 alias TOKalias = TOK.TOKalias; 475 alias TOKoverride = TOK.TOKoverride; 476 alias TOKdelegate = TOK.TOKdelegate; 477 alias TOKfunction = TOK.TOKfunction; 478 alias TOKmixin = TOK.TOKmixin; 479 alias TOKalign = TOK.TOKalign; 480 alias TOKextern = TOK.TOKextern; 481 alias TOKprivate = TOK.TOKprivate; 482 alias TOKprotected = TOK.TOKprotected; 483 alias TOKpublic = TOK.TOKpublic; 484 alias TOKexport = TOK.TOKexport; 485 alias TOKstatic = TOK.TOKstatic; 486 alias TOKfinal = TOK.TOKfinal; 487 alias TOKconst = TOK.TOKconst; 488 alias TOKabstract = TOK.TOKabstract; 489 alias TOKdebug = TOK.TOKdebug; 490 alias TOKdeprecated = TOK.TOKdeprecated; 491 alias TOKin = TOK.TOKin; 492 alias TOKout = TOK.TOKout; 493 alias TOKinout = TOK.TOKinout; 494 alias TOKlazy = TOK.TOKlazy; 495 alias TOKauto = TOK.TOKauto; 496 alias TOKpackage = TOK.TOKpackage; 497 alias TOKmanifest = TOK.TOKmanifest; 498 alias TOKimmutable = TOK.TOKimmutable; 499 alias TOKif = TOK.TOKif; 500 alias TOKelse = TOK.TOKelse; 501 alias TOKwhile = TOK.TOKwhile; 502 alias TOKfor = TOK.TOKfor; 503 alias TOKdo = TOK.TOKdo; 504 alias TOKswitch = TOK.TOKswitch; 505 alias TOKcase = TOK.TOKcase; 506 alias TOKdefault = TOK.TOKdefault; 507 alias TOKbreak = TOK.TOKbreak; 508 alias TOKcontinue = TOK.TOKcontinue; 509 alias TOKwith = TOK.TOKwith; 510 alias TOKsynchronized = TOK.TOKsynchronized; 511 alias TOKreturn = TOK.TOKreturn; 512 alias TOKgoto = TOK.TOKgoto; 513 alias TOKtry = TOK.TOKtry; 514 alias TOKcatch = TOK.TOKcatch; 515 alias TOKfinally = TOK.TOKfinally; 516 alias TOKasm = TOK.TOKasm; 517 alias TOKforeach = TOK.TOKforeach; 518 alias TOKforeach_reverse = TOK.TOKforeach_reverse; 519 alias TOKscope = TOK.TOKscope; 520 alias TOKon_scope_exit = TOK.TOKon_scope_exit; 521 alias TOKon_scope_failure = TOK.TOKon_scope_failure; 522 alias TOKon_scope_success = TOK.TOKon_scope_success; 523 alias TOKbody = TOK.TOKbody; 524 alias TOKinvariant = TOK.TOKinvariant; 525 alias TOKunittest = TOK.TOKunittest; 526 alias TOKargTypes = TOK.TOKargTypes; 527 alias TOKref = TOK.TOKref; 528 alias TOKmacro = TOK.TOKmacro; 529 alias TOKparameters = TOK.TOKparameters; 530 alias TOKtraits = TOK.TOKtraits; 531 alias TOKoverloadset = TOK.TOKoverloadset; 532 alias TOKpure = TOK.TOKpure; 533 alias TOKnothrow = TOK.TOKnothrow; 534 alias TOKgshared = TOK.TOKgshared; 535 alias TOKline = TOK.TOKline; 536 alias TOKfile = TOK.TOKfile; 537 alias TOKfilefullpath = TOK.TOKfilefullpath; 538 alias TOKmodulestring = TOK.TOKmodulestring; 539 alias TOKfuncstring = TOK.TOKfuncstring; 540 alias TOKprettyfunc = TOK.TOKprettyfunc; 541 alias TOKshared = TOK.TOKshared; 542 alias TOKat = TOK.TOKat; 543 alias TOKpow = TOK.TOKpow; 544 alias TOKpowass = TOK.TOKpowass; 545 alias TOKgoesto = TOK.TOKgoesto; 546 alias TOKvector = TOK.TOKvector; 547 alias TOKpound = TOK.TOKpound; 548 alias TOKinterval = TOK.TOKinterval; 549 alias TOKvoidexp = TOK.TOKvoidexp; 550 alias TOKcantexp = TOK.TOKcantexp; 551 alias TOKMAX = TOK.TOKMAX; 552 553 enum TOKwild = TOKinout; 554 555 /*********************************************************** 556 */ 557 extern (C++) struct Token 558 { 559 Token* next; 560 Loc loc; 561 const(char)* ptr; // pointer to first character of this token within buffer 562 TOK value; 563 const(char)* blockComment; // doc comment string prior to this token 564 const(char)* lineComment; // doc comment for previous token 565 566 union 567 { 568 // Integers 569 d_int64 int64value; 570 d_uns64 uns64value; 571 // Floats 572 real_t floatvalue; 573 574 struct 575 { 576 const(char)* ustring; // UTF8 string 577 uint len; 578 ubyte postfix; // 'c', 'w', 'd' 579 } 580 581 Identifier ident; 582 } 583 584 extern (D) private __gshared immutable string[TOKMAX] tochars = 585 [ 586 // Keywords 587 TOKthis: "this", 588 TOKsuper: "super", 589 TOKassert: "assert", 590 TOKnull: "null", 591 TOKtrue: "true", 592 TOKfalse: "false", 593 TOKcast: "cast", 594 TOKnew: "new", 595 TOKdelete: "delete", 596 TOKthrow: "throw", 597 TOKmodule: "module", 598 TOKpragma: "pragma", 599 TOKtypeof: "typeof", 600 TOKtypeid: "typeid", 601 TOKtemplate: "template", 602 TOKvoid: "void", 603 TOKint8: "byte", 604 TOKuns8: "ubyte", 605 TOKint16: "short", 606 TOKuns16: "ushort", 607 TOKint32: "int", 608 TOKuns32: "uint", 609 TOKint64: "long", 610 TOKuns64: "ulong", 611 TOKint128: "cent", 612 TOKuns128: "ucent", 613 TOKfloat32: "float", 614 TOKfloat64: "double", 615 TOKfloat80: "real", 616 TOKbool: "bool", 617 TOKchar: "char", 618 TOKwchar: "wchar", 619 TOKdchar: "dchar", 620 TOKimaginary32: "ifloat", 621 TOKimaginary64: "idouble", 622 TOKimaginary80: "ireal", 623 TOKcomplex32: "cfloat", 624 TOKcomplex64: "cdouble", 625 TOKcomplex80: "creal", 626 TOKdelegate: "delegate", 627 TOKfunction: "function", 628 TOKis: "is", 629 TOKif: "if", 630 TOKelse: "else", 631 TOKwhile: "while", 632 TOKfor: "for", 633 TOKdo: "do", 634 TOKswitch: "switch", 635 TOKcase: "case", 636 TOKdefault: "default", 637 TOKbreak: "break", 638 TOKcontinue: "continue", 639 TOKsynchronized: "synchronized", 640 TOKreturn: "return", 641 TOKgoto: "goto", 642 TOKtry: "try", 643 TOKcatch: "catch", 644 TOKfinally: "finally", 645 TOKwith: "with", 646 TOKasm: "asm", 647 TOKforeach: "foreach", 648 TOKforeach_reverse: "foreach_reverse", 649 TOKscope: "scope", 650 TOKstruct: "struct", 651 TOKclass: "class", 652 TOKinterface: "interface", 653 TOKunion: "union", 654 TOKenum: "enum", 655 TOKimport: "import", 656 TOKmixin: "mixin", 657 TOKstatic: "static", 658 TOKfinal: "final", 659 TOKconst: "const", 660 TOKalias: "alias", 661 TOKoverride: "override", 662 TOKabstract: "abstract", 663 TOKdebug: "debug", 664 TOKdeprecated: "deprecated", 665 TOKin: "in", 666 TOKout: "out", 667 TOKinout: "inout", 668 TOKlazy: "lazy", 669 TOKauto: "auto", 670 TOKalign: "align", 671 TOKextern: "extern", 672 TOKprivate: "private", 673 TOKpackage: "package", 674 TOKprotected: "protected", 675 TOKpublic: "public", 676 TOKexport: "export", 677 TOKbody: "body", 678 TOKinvariant: "invariant", 679 TOKunittest: "unittest", 680 TOKversion: "version", 681 TOKargTypes: "__argTypes", 682 TOKparameters: "__parameters", 683 TOKref: "ref", 684 TOKmacro: "macro", 685 TOKpure: "pure", 686 TOKnothrow: "nothrow", 687 TOKgshared: "__gshared", 688 TOKtraits: "__traits", 689 TOKvector: "__vector", 690 TOKoverloadset: "__overloadset", 691 TOKfile: "__FILE__", 692 TOKfilefullpath: "__FILE_FULL_PATH__", 693 TOKline: "__LINE__", 694 TOKmodulestring: "__MODULE__", 695 TOKfuncstring: "__FUNCTION__", 696 TOKprettyfunc: "__PRETTY_FUNCTION__", 697 TOKshared: "shared", 698 TOKimmutable: "immutable", 699 700 TOKeof: "EOF", 701 TOKlcurly: "{", 702 TOKrcurly: "}", 703 TOKlparen: "(", 704 TOKrparen: ")", 705 TOKlbracket: "[", 706 TOKrbracket: "]", 707 TOKsemicolon: ";", 708 TOKcolon: ":", 709 TOKcomma: ",", 710 TOKdot: ".", 711 TOKxor: "^", 712 TOKxorass: "^=", 713 TOKassign: "=", 714 TOKconstruct: "=", 715 TOKblit: "=", 716 TOKlt: "<", 717 TOKgt: ">", 718 TOKle: "<=", 719 TOKge: ">=", 720 TOKequal: "==", 721 TOKnotequal: "!=", 722 TOKunord: "!<>=", 723 TOKue: "!<>", 724 TOKlg: "<>", 725 TOKleg: "<>=", 726 TOKule: "!>", 727 TOKul: "!>=", 728 TOKuge: "!<", 729 TOKug: "!<=", 730 TOKnot: "!", 731 TOKshl: "<<", 732 TOKshr: ">>", 733 TOKushr: ">>>", 734 TOKadd: "+", 735 TOKmin: "-", 736 TOKmul: "*", 737 TOKdiv: "/", 738 TOKmod: "%", 739 TOKslice: "..", 740 TOKdotdotdot: "...", 741 TOKand: "&", 742 TOKandand: "&&", 743 TOKor: "|", 744 TOKoror: "||", 745 TOKarray: "[]", 746 TOKindex: "[i]", 747 TOKaddress: "&", 748 TOKstar: "*", 749 TOKtilde: "~", 750 TOKdollar: "$", 751 TOKplusplus: "++", 752 TOKminusminus: "--", 753 TOKpreplusplus: "++", 754 TOKpreminusminus: "--", 755 TOKtype: "type", 756 TOKquestion: "?", 757 TOKneg: "-", 758 TOKuadd: "+", 759 TOKvar: "var", 760 TOKaddass: "+=", 761 TOKminass: "-=", 762 TOKmulass: "*=", 763 TOKdivass: "/=", 764 TOKmodass: "%=", 765 TOKshlass: "<<=", 766 TOKshrass: ">>=", 767 TOKushrass: ">>>=", 768 TOKandass: "&=", 769 TOKorass: "|=", 770 TOKcatass: "~=", 771 TOKcat: "~", 772 TOKcall: "call", 773 TOKidentity: "is", 774 TOKnotidentity: "!is", 775 TOKidentifier: "identifier", 776 TOKat: "@", 777 TOKpow: "^^", 778 TOKpowass: "^^=", 779 TOKgoesto: "=>", 780 TOKpound: "#", 781 782 // For debugging 783 TOKerror: "error", 784 TOKdotid: "dotid", 785 TOKdottd: "dottd", 786 TOKdotti: "dotti", 787 TOKdotvar: "dotvar", 788 TOKdottype: "dottype", 789 TOKsymoff: "symoff", 790 TOKarraylength: "arraylength", 791 TOKarrayliteral: "arrayliteral", 792 TOKassocarrayliteral: "assocarrayliteral", 793 TOKstructliteral: "structliteral", 794 TOKstring: "string", 795 TOKdsymbol: "symbol", 796 TOKtuple: "tuple", 797 TOKdeclaration: "declaration", 798 TOKon_scope_exit: "scope(exit)", 799 TOKon_scope_success: "scope(success)", 800 TOKon_scope_failure: "scope(failure)", 801 TOKdelegateptr: "delegateptr", 802 803 // Finish up 804 TOKreserved: "reserved", 805 TOKremove: "remove", 806 TOKnewanonclass: "newanonclass", 807 TOKcomment: "comment", 808 TOKclassreference: "classreference", 809 TOKthrownexception: "thrownexception", 810 TOKdelegatefuncptr: "delegatefuncptr", 811 TOKarrow: "arrow", 812 TOKint32v: "int32v", 813 TOKuns32v: "uns32v", 814 TOKint64v: "int64v", 815 TOKuns64v: "uns64v", 816 TOKint128v: "int128v", 817 TOKuns128v: "uns128v", 818 TOKfloat32v: "float32v", 819 TOKfloat64v: "float64v", 820 TOKfloat80v: "float80v", 821 TOKimaginary32v: "imaginary32v", 822 TOKimaginary64v: "imaginary64v", 823 TOKimaginary80v: "imaginary80v", 824 TOKcharv: "charv", 825 TOKwcharv: "wcharv", 826 TOKdcharv: "dcharv", 827 828 TOKhalt: "halt", 829 TOKxstring: "xstring", 830 TOKmanifest: "manifest", 831 832 TOKinterval: "interval", 833 TOKvoidexp: "voidexp", 834 TOKcantexp: "cantexp", 835 ]; 836 837 static this() 838 { 839 Identifier.initTable(); 840 foreach (kw; keywords) 841 { 842 //printf("keyword[%d] = '%s'\n",kw, tochars[kw].ptr); 843 Identifier.idPool(tochars[kw].ptr, tochars[kw].length, cast(uint)kw); 844 } 845 846 foreach (i, s; tochars) 847 { 848 assert(s.length); 849 } 850 } 851 852 __gshared Token* freelist = null; 853 854 static Token* alloc() 855 { 856 if (Token.freelist) 857 { 858 Token* t = freelist; 859 freelist = t.next; 860 t.next = null; 861 return t; 862 } 863 return new Token(); 864 } 865 866 void free() 867 { 868 next = freelist; 869 freelist = &this; 870 } 871 872 int isKeyword() const 873 { 874 foreach (kw; keywords) 875 { 876 if (kw == value) 877 return 1; 878 } 879 return 0; 880 } 881 882 debug 883 { 884 void print() 885 { 886 fprintf(stderr, "%s\n", toChars()); 887 } 888 } 889 890 /**** 891 * Set to contents of ptr[0..length] 892 * Params: 893 * ptr = pointer to string 894 * length = length of string 895 */ 896 final void setString(const(char)* ptr, size_t length) 897 { 898 auto s = cast(char*)mem.xmalloc(length + 1); 899 memcpy(s, ptr, length); 900 s[length] = 0; 901 ustring = s; 902 len = cast(uint)length; 903 postfix = 0; 904 } 905 906 /**** 907 * Set to contents of buf 908 * Params: 909 * buf = string (not zero terminated) 910 */ 911 final void setString(const ref OutBuffer buf) 912 { 913 setString(cast(const(char)*)buf.data, buf.offset); 914 } 915 916 /**** 917 * Set to empty string 918 */ 919 final void setString() 920 { 921 ustring = ""; 922 len = 0; 923 postfix = 0; 924 } 925 926 extern (C++) const(char)* toChars() const 927 { 928 __gshared char[3 + 3 * floatvalue.sizeof + 1] buffer; 929 const(char)* p = &buffer[0]; 930 switch (value) 931 { 932 case TOKint32v: 933 sprintf(&buffer[0], "%d", cast(d_int32)int64value); 934 break; 935 case TOKuns32v: 936 case TOKcharv: 937 case TOKwcharv: 938 case TOKdcharv: 939 sprintf(&buffer[0], "%uU", cast(d_uns32)uns64value); 940 break; 941 case TOKint64v: 942 sprintf(&buffer[0], "%lldL", cast(long)int64value); 943 break; 944 case TOKuns64v: 945 sprintf(&buffer[0], "%lluUL", cast(ulong)uns64value); 946 break; 947 case TOKfloat32v: 948 CTFloat.sprint(&buffer[0], 'g', floatvalue); 949 strcat(&buffer[0], "f"); 950 break; 951 case TOKfloat64v: 952 CTFloat.sprint(&buffer[0], 'g', floatvalue); 953 break; 954 case TOKfloat80v: 955 CTFloat.sprint(&buffer[0], 'g', floatvalue); 956 strcat(&buffer[0], "L"); 957 break; 958 case TOKimaginary32v: 959 CTFloat.sprint(&buffer[0], 'g', floatvalue); 960 strcat(&buffer[0], "fi"); 961 break; 962 case TOKimaginary64v: 963 CTFloat.sprint(&buffer[0], 'g', floatvalue); 964 strcat(&buffer[0], "i"); 965 break; 966 case TOKimaginary80v: 967 CTFloat.sprint(&buffer[0], 'g', floatvalue); 968 strcat(&buffer[0], "Li"); 969 break; 970 case TOKstring: 971 { 972 OutBuffer buf; 973 buf.writeByte('"'); 974 for (size_t i = 0; i < len;) 975 { 976 dchar c; 977 utf_decodeChar(ustring, len, i, c); 978 switch (c) 979 { 980 case 0: 981 break; 982 case '"': 983 case '\\': 984 buf.writeByte('\\'); 985 goto default; 986 default: 987 if (c <= 0x7F) 988 { 989 if (isprint(c)) 990 buf.writeByte(c); 991 else 992 buf.printf("\\x%02x", c); 993 } 994 else if (c <= 0xFFFF) 995 buf.printf("\\u%04x", c); 996 else 997 buf.printf("\\U%08x", c); 998 continue; 999 } 1000 break; 1001 } 1002 buf.writeByte('"'); 1003 if (postfix) 1004 buf.writeByte(postfix); 1005 p = buf.extractString(); 1006 } 1007 break; 1008 case TOKxstring: 1009 { 1010 OutBuffer buf; 1011 buf.writeByte('x'); 1012 buf.writeByte('"'); 1013 foreach (size_t i; 0 .. len) 1014 { 1015 if (i) 1016 buf.writeByte(' '); 1017 buf.printf("%02x", ustring[i]); 1018 } 1019 buf.writeByte('"'); 1020 if (postfix) 1021 buf.writeByte(postfix); 1022 buf.writeByte(0); 1023 p = buf.extractData(); 1024 break; 1025 } 1026 case TOKidentifier: 1027 case TOKenum: 1028 case TOKstruct: 1029 case TOKimport: 1030 case TOKwchar: 1031 case TOKdchar: 1032 case TOKbool: 1033 case TOKchar: 1034 case TOKint8: 1035 case TOKuns8: 1036 case TOKint16: 1037 case TOKuns16: 1038 case TOKint32: 1039 case TOKuns32: 1040 case TOKint64: 1041 case TOKuns64: 1042 case TOKint128: 1043 case TOKuns128: 1044 case TOKfloat32: 1045 case TOKfloat64: 1046 case TOKfloat80: 1047 case TOKimaginary32: 1048 case TOKimaginary64: 1049 case TOKimaginary80: 1050 case TOKcomplex32: 1051 case TOKcomplex64: 1052 case TOKcomplex80: 1053 case TOKvoid: 1054 p = ident.toChars(); 1055 break; 1056 default: 1057 p = toChars(value); 1058 break; 1059 } 1060 return p; 1061 } 1062 1063 static const(char)* toChars(TOK value) 1064 { 1065 return toString(value).ptr; 1066 } 1067 1068 extern (D) static string toString(TOK value) pure nothrow @nogc @safe 1069 { 1070 return tochars[value]; 1071 } 1072 } 1073 1074 /**************************************** 1075 */ 1076 1077 private immutable TOK[] keywords = 1078 [ 1079 TOKthis, 1080 TOKsuper, 1081 TOKassert, 1082 TOKnull, 1083 TOKtrue, 1084 TOKfalse, 1085 TOKcast, 1086 TOKnew, 1087 TOKdelete, 1088 TOKthrow, 1089 TOKmodule, 1090 TOKpragma, 1091 TOKtypeof, 1092 TOKtypeid, 1093 TOKtemplate, 1094 TOKvoid, 1095 TOKint8, 1096 TOKuns8, 1097 TOKint16, 1098 TOKuns16, 1099 TOKint32, 1100 TOKuns32, 1101 TOKint64, 1102 TOKuns64, 1103 TOKint128, 1104 TOKuns128, 1105 TOKfloat32, 1106 TOKfloat64, 1107 TOKfloat80, 1108 TOKbool, 1109 TOKchar, 1110 TOKwchar, 1111 TOKdchar, 1112 TOKimaginary32, 1113 TOKimaginary64, 1114 TOKimaginary80, 1115 TOKcomplex32, 1116 TOKcomplex64, 1117 TOKcomplex80, 1118 TOKdelegate, 1119 TOKfunction, 1120 TOKis, 1121 TOKif, 1122 TOKelse, 1123 TOKwhile, 1124 TOKfor, 1125 TOKdo, 1126 TOKswitch, 1127 TOKcase, 1128 TOKdefault, 1129 TOKbreak, 1130 TOKcontinue, 1131 TOKsynchronized, 1132 TOKreturn, 1133 TOKgoto, 1134 TOKtry, 1135 TOKcatch, 1136 TOKfinally, 1137 TOKwith, 1138 TOKasm, 1139 TOKforeach, 1140 TOKforeach_reverse, 1141 TOKscope, 1142 TOKstruct, 1143 TOKclass, 1144 TOKinterface, 1145 TOKunion, 1146 TOKenum, 1147 TOKimport, 1148 TOKmixin, 1149 TOKstatic, 1150 TOKfinal, 1151 TOKconst, 1152 TOKalias, 1153 TOKoverride, 1154 TOKabstract, 1155 TOKdebug, 1156 TOKdeprecated, 1157 TOKin, 1158 TOKout, 1159 TOKinout, 1160 TOKlazy, 1161 TOKauto, 1162 TOKalign, 1163 TOKextern, 1164 TOKprivate, 1165 TOKpackage, 1166 TOKprotected, 1167 TOKpublic, 1168 TOKexport, 1169 TOKbody, 1170 TOKinvariant, 1171 TOKunittest, 1172 TOKversion, 1173 TOKargTypes, 1174 TOKparameters, 1175 TOKref, 1176 TOKmacro, 1177 TOKpure, 1178 TOKnothrow, 1179 TOKgshared, 1180 TOKtraits, 1181 TOKvector, 1182 TOKoverloadset, 1183 TOKfile, 1184 TOKfilefullpath, 1185 TOKline, 1186 TOKmodulestring, 1187 TOKfuncstring, 1188 TOKprettyfunc, 1189 TOKshared, 1190 TOKimmutable, 1191 ];