package syntax
import (
"bytes"
"encoding/binary"
"fmt"
"slices"
"sort"
"strings"
"unicode"
"unicode/utf8"
)
type CharSet struct {
ranges []SingleRange
categories []Category
sub *CharSet
negate bool
anything bool
ascii *asciiBitmap
}
type asciiBitmap struct {
bits [2 ]uint64
}
type Category struct {
Negate bool
Cat string
}
type SingleRange struct {
First rune
Last rune
}
const (
SpaceCategoryText = " "
WordCategoryText = "W"
)
var (
ecmaSpace = []rune {0x0009 , 0x000e , 0x0020 , 0x0021 , 0x00a0 , 0x00a1 , 0x1680 , 0x1681 , 0x2000 , 0x200b , 0x2028 , 0x202a , 0x202f , 0x2030 , 0x205f , 0x2060 , 0x3000 , 0x3001 , 0xfeff , 0xff00 }
ecmaWord = []rune {0x0030 , 0x003a , 0x0041 , 0x005b , 0x005f , 0x0060 , 0x0061 , 0x007b }
ecmaDigit = []rune {0x0030 , 0x003a }
re2Space = []rune {0x0009 , 0x000b , 0x000c , 0x000e , 0x0020 , 0x0021 }
)
var (
AnyClass = getCharSetFromOldString ([]rune {0 }, false )
ECMAAnyClass = getCharSetFromOldString ([]rune {0 , 0x000a , 0x000b , 0x000d , 0x000e }, false )
NoneClass = getCharSetFromOldString (nil , false )
ECMAWordClass = getCharSetFromOldString (ecmaWord , false )
NotECMAWordClass = getCharSetFromOldString (ecmaWord , true )
ECMASpaceClass = getCharSetFromOldString (ecmaSpace , false )
NotECMASpaceClass = getCharSetFromOldString (ecmaSpace , true )
ECMADigitClass = getCharSetFromOldString (ecmaDigit , false )
NotECMADigitClass = getCharSetFromOldString (ecmaDigit , true )
WordClass = getCharSetFromCategoryString (false , false , WordCategoryText )
NotWordClass = getCharSetFromCategoryString (true , false , WordCategoryText )
SpaceClass = getCharSetFromCategoryString (false , false , SpaceCategoryText )
NotSpaceClass = getCharSetFromCategoryString (true , false , SpaceCategoryText )
DigitClass = getCharSetFromCategoryString (false , false , "Nd" )
NotDigitClass = getCharSetFromCategoryString (false , true , "Nd" )
RE2SpaceClass = getCharSetFromOldString (re2Space , false )
NotRE2SpaceClass = getCharSetFromOldString (re2Space , true )
NotNewLineClass = getCharSetFromOldString ([]rune {0x0a , 0x0b }, true )
)
var unicodeCategories = func () map [string ]*unicode .RangeTable {
retVal := make (map [string ]*unicode .RangeTable )
for k , v := range unicode .Scripts {
retVal [k ] = v
}
for k , v := range unicode .Categories {
retVal [k ] = v
}
for k , v := range unicode .CategoryAliases {
retVal [k ] = unicode .Categories [v ]
}
for k , v := range unicode .Properties {
retVal [k ] = v
}
for k , v := range unicodeAliasCategories {
retVal [k ] = v
}
return retVal
}()
func getCharSetFromCategoryString(negateSet bool , negateCat bool , cats ...string ) func () *CharSet {
if negateCat && negateSet {
panic ("BUG! You should only negate the set OR the category in a constant setup, but not both" )
}
c := CharSet {negate : negateSet }
c .categories = make ([]Category , len (cats ))
for i , cat := range cats {
c .categories [i ] = Category {Cat : cat , Negate : negateCat }
}
return func () *CharSet {
local := c
return &local
}
}
func getCharSetFromOldString(setText []rune , negate bool ) func () *CharSet {
c := CharSet {}
if len (setText ) > 0 {
fillFirst := false
l := len (setText )
if negate {
if setText [0 ] == 0 {
setText = setText [1 :]
} else {
l ++
fillFirst = true
}
}
if l %2 == 0 {
c .ranges = make ([]SingleRange , l /2 )
} else {
c .ranges = make ([]SingleRange , l /2 +1 )
}
first := true
if fillFirst {
c .ranges [0 ] = SingleRange {First : 0 }
first = false
}
i := 0
for _ , r := range setText {
if first {
c .ranges [i ] = SingleRange {First : r }
first = false
} else {
c .ranges [i ].Last = r - 1
i ++
first = true
}
}
if !first {
c .ranges [i ].Last = utf8 .MaxRune
}
if len (c .ranges ) == 1 && c .ranges [0 ].First == 0 && c .ranges [0 ].Last >= unicode .MaxRune {
c .anything = !negate
}
}
return func () *CharSet {
local := c
return &local
}
}
func (c CharSet ) Copy () CharSet {
ret := CharSet {
anything : c .anything ,
negate : c .negate ,
}
ret .ranges = append (ret .ranges , c .ranges ...)
ret .categories = append (ret .categories , c .categories ...)
if c .sub != nil {
sub := c .sub .Copy ()
ret .sub = &sub
}
return ret
}
func (c CharSet ) String () string {
buf := &bytes .Buffer {}
buf .WriteRune ('[' )
if c .IsNegated () {
buf .WriteRune ('^' )
}
for _ , r := range c .ranges {
buf .WriteString (CharDescription (r .First ))
if r .First != r .Last {
if r .Last -r .First != 1 {
buf .WriteRune ('-' )
}
buf .WriteString (CharDescription (r .Last ))
}
}
for _ , c := range c .categories {
buf .WriteString (c .String ())
}
if c .sub != nil {
buf .WriteRune ('-' )
buf .WriteString (c .sub .String ())
}
buf .WriteRune (']' )
return buf .String ()
}
func b2i(b bool ) byte {
if b {
return 1
}
return 0
}
func (c CharSet ) mapHashFill (buf *bytes .Buffer ) {
buf .WriteByte (b2i (c .negate ) + b2i (c .anything )*2 )
_ = binary .Write (buf , binary .LittleEndian , int32 (len (c .ranges )))
_ = binary .Write (buf , binary .LittleEndian , int32 (len (c .categories )))
for _ , r := range c .ranges {
buf .WriteRune (r .First )
buf .WriteRune (r .Last )
}
for _ , ct := range c .categories {
if ct .Negate {
_ = binary .Write (buf , binary .LittleEndian , int8 (-1 *len (ct .Cat )))
} else {
_ = binary .Write (buf , binary .LittleEndian , int8 (len (ct .Cat )))
}
buf .WriteString (ct .Cat )
}
if c .sub != nil {
c .sub .mapHashFill (buf )
}
}
func NewCharSetRuntime (buf string ) CharSet {
retVal := CharSet {}
b := bytes .NewBufferString (buf )
val , _ := b .ReadByte ()
retVal .negate = (val &0x1 == 0x1 )
retVal .anything = (val &0x2 == 0x2 )
var lenRanges , lenCats int32
_ = binary .Read (b , binary .LittleEndian , &lenRanges )
_ = binary .Read (b , binary .LittleEndian , &lenCats )
retVal .ranges = make ([]SingleRange , lenRanges )
for i := 0 ; i < int (lenRanges ); i ++ {
r := SingleRange {}
r .First , _, _ = b .ReadRune ()
r .Last , _, _ = b .ReadRune ()
retVal .ranges [i ] = r
}
retVal .categories = make ([]Category , lenCats )
for i := 0 ; i < int (lenCats ); i ++ {
var lenCat int8
c := Category {}
_ = binary .Read (b , binary .LittleEndian , &lenCat )
if lenCat < 0 {
c .Negate = true
lenCat *= -1
}
c .Cat = string (b .Next (int (lenCat )))
retVal .categories [i ] = c
}
if b .Len () > 0 {
sub := NewCharSetRuntime (b .String ())
retVal .sub = &sub
}
return retVal
}
func (c CharSet ) CharIn (ch rune ) bool {
if ch >= 0 && ch < 128 && c .ascii != nil {
return (c .ascii .bits [ch /64 ] & (1 << (uint (ch ) % 64 ))) != 0
}
return c .charInSlow (ch )
}
func (c CharSet ) charInSlow (ch rune ) bool {
val := false
n := len (c .ranges )
if n > 0 {
if n <= 4 {
for _ , r := range c .ranges {
if ch < r .First {
break
}
if ch <= r .Last {
val = true
break
}
}
} else {
lo , hi := 0 , n
for lo < hi {
mid := int (uint (lo +hi ) >> 1 )
if c .ranges [mid ].First <= ch {
lo = mid + 1
} else {
hi = mid
}
}
if lo > 0 && ch <= c .ranges [lo -1 ].Last {
val = true
}
}
}
if !val && len (c .categories ) > 0 {
val = c .charInCategories (ch )
}
if c .negate {
val = !val
}
if val && c .sub != nil {
val = !c .sub .CharIn (ch )
}
return val
}
func (c *CharSet ) prepareASCIIBitmap () {
if c == nil || c .ascii != nil {
return
}
if c .sub != nil {
c .sub .prepareASCIIBitmap ()
}
bm := &asciiBitmap {}
for i := range rune (128 ) {
if c .charInSlow (i ) {
bm .bits [i /64 ] |= 1 << (uint (i ) % 64 )
}
}
c .ascii = bm
}
func (c *CharSet ) charInCategories (ch rune ) bool {
for _ , ct := range c .categories {
if ct .Cat == SpaceCategoryText {
if unicode .IsSpace (ch ) {
return !ct .Negate
} else if ct .Negate {
return true
}
} else if ct .Cat == WordCategoryText {
if IsWordChar (ch ) {
return !ct .Negate
} else if ct .Negate {
return true
}
} else if unicode .Is (unicodeCategories [ct .Cat ], ch ) {
return !ct .Negate
} else if ct .Negate {
return true
}
}
return false
}
func (c Category ) String () string {
switch c .Cat {
case SpaceCategoryText :
if c .Negate {
return "\\S"
}
return "\\s"
case WordCategoryText :
if c .Negate {
return "\\W"
}
return "\\w"
}
if _ , ok := unicodeCategories [c .Cat ]; ok {
if c .Negate {
return "\\P{" + c .Cat + "}"
}
return "\\p{" + c .Cat + "}"
}
return "Unknown category: " + c .Cat
}
func CharDescription (ch rune ) string {
b := &bytes .Buffer {}
escape (b , ch , false )
return b .String ()
}
func IsWordChar (r rune ) bool {
return unicode .In (r ,
unicode .Categories ["L" ], unicode .Categories ["Mn" ],
unicode .Categories ["Nd" ], unicode .Categories ["Pc" ]) || r == '\u200D' || r == '\u200C'
}
func IsECMAWordChar (r rune ) bool {
return unicode .In (r ,
unicode .Categories ["L" ], unicode .Categories ["Mn" ],
unicode .Categories ["Nd" ], unicode .Categories ["Pc" ])
}
func IsECMAIdentifierStartChar (r rune ) bool {
return r == '$' || r == '_' || unicode .In (r , unicode .L , unicode .Nl , unicode .Other_ID_Start )
}
func IsECMAIdentifierChar (r rune ) bool {
return IsECMAIdentifierStartChar (r ) || r == '\u200C' || r == '\u200D' ||
unicode .In (r , unicode .Mn , unicode .Mc , unicode .Nd , unicode .Pc , unicode .Other_ID_Continue )
}
func (c CharSet ) SingletonChar () rune {
return c .ranges [0 ].First
}
func (c CharSet ) IsSingleton () bool {
return !c .negate &&
len (c .categories ) == 0 && len (c .ranges ) == 1 &&
c .sub == nil &&
c .ranges [0 ].First == c .ranges [0 ].Last
}
func (c CharSet ) IsSingletonInverse () bool {
return c .negate &&
len (c .categories ) == 0 && len (c .ranges ) == 1 &&
c .sub == nil &&
c .ranges [0 ].First == c .ranges [0 ].Last
}
func (c CharSet ) IsMergeable () bool {
return !c .IsNegated () && !c .HasSubtraction ()
}
func (c CharSet ) IsNegated () bool {
return c .negate
}
func (c CharSet ) HasSubtraction () bool {
return c .sub != nil
}
func (c CharSet ) IsEmpty () bool {
return len (c .ranges ) == 0 && len (c .categories ) == 0 && c .sub == nil
}
func (c CharSet ) IsAnything () bool {
return c .anything
}
func (c *CharSet ) addDigit (ecma , negate bool ) {
if ecma {
if negate {
c .addRanges (NotECMADigitClass ().ranges )
} else {
c .addRanges (ECMADigitClass ().ranges )
}
} else {
c .addCategories (Category {Cat : "Nd" , Negate : negate })
}
}
func (c *CharSet ) addChar (ch rune ) {
c .addRange (ch , ch )
}
func (c *CharSet ) addSpace (ecma , re2 , negate bool ) {
if ecma {
if negate {
c .addRanges (NotECMASpaceClass ().ranges )
} else {
c .addRanges (ECMASpaceClass ().ranges )
}
} else if re2 {
if negate {
c .addRanges (NotRE2SpaceClass ().ranges )
} else {
c .addRanges (RE2SpaceClass ().ranges )
}
} else {
c .addCategories (Category {Cat : SpaceCategoryText , Negate : negate })
}
}
func (c *CharSet ) addWord (ecma , negate bool ) {
if ecma {
if negate {
c .addRanges (NotECMAWordClass ().ranges )
} else {
c .addRanges (ECMAWordClass ().ranges )
}
} else {
c .addCategories (Category {Cat : WordCategoryText , Negate : negate })
}
}
func (c *CharSet ) addSet (set CharSet ) {
if c .anything {
return
}
if set .anything {
c .makeAnything ()
return
}
c .ranges = append (c .ranges , set .ranges ...)
c .addCategories (set .categories ...)
c .canonicalize ()
}
func (c *CharSet ) makeAnything () {
c .anything = true
c .categories = []Category {}
c .ranges = []SingleRange {{First : 0 , Last : unicode .MaxRune }}
}
func (c *CharSet ) addCategories (cats ...Category ) {
if c .anything {
return
}
for _ , ct := range cats {
found := false
for _ , ct2 := range c .categories {
if ct .Cat == ct2 .Cat {
if ct .Negate != ct2 .Negate {
c .makeAnything ()
return
}
found = true
break
}
}
if !found {
c .categories = append (c .categories , ct )
}
}
}
func (c *CharSet ) addRanges (ranges []SingleRange ) {
if c .anything {
return
}
c .ranges = append (c .ranges , ranges ...)
c .canonicalize ()
}
func (c *CharSet ) addNegativeRanges (ranges []SingleRange ) {
if c .anything {
return
}
var hi rune
for _ , r := range ranges {
if hi < r .First {
c .ranges = append (c .ranges , SingleRange {hi , r .First - 1 })
}
hi = r .Last + 1
}
if hi < utf8 .MaxRune {
c .ranges = append (c .ranges , SingleRange {hi , utf8 .MaxRune })
}
c .canonicalize ()
}
func normalizeUnicodeCategoryAlias(catName string ) string {
var b strings .Builder
b .Grow (len (catName ))
for _ , ch := range catName {
switch ch {
case '_' , '-' , ' ' :
continue
default :
b .WriteRune (unicode .ToLower (ch ))
}
}
return b .String ()
}
func canonicalUnicodeCatName(catName string ) (string , bool ) {
if _ , ok := unicodeCategories [catName ]; ok {
return catName , true
}
normalized := normalizeUnicodeCategoryAlias (catName )
if canonical , ok := unicodeSupportedPropertyAliases [normalized ]; ok {
return canonical , true
}
if canonical , ok := unicodeBarePropertyValueAliases [normalized ]; ok {
return canonical , true
}
if eq := strings .IndexRune (catName , '=' ); eq >= 0 {
propName := catName [:eq ]
valueName := catName [eq +1 :]
prop , ok := unicodeSupportedPropertyAliases [normalizeUnicodeCategoryAlias (propName )]
if !ok {
return "" , false
}
values := unicodeSupportedPropertyValueAliases [prop ]
if values == nil {
return "" , false
}
value , ok := values [normalizeUnicodeCategoryAlias (valueName )]
if !ok {
return "" , false
}
canonical := prop + "=" + value
if _ , ok := unicodeCategories [canonical ]; ok {
return canonical , true
}
}
return "" , false
}
func (c *CharSet ) addCategory (categoryName string , negate , caseInsensitive bool ) {
var ok bool
categoryName , ok = canonicalUnicodeCatName (categoryName )
if !ok {
panic (fmt .Errorf ("unknown unicode category, script, or property '%v'" , categoryName ))
}
if caseInsensitive && (categoryName == "Ll" || categoryName == "Lu" || categoryName == "Lt" ) {
c .addCategories (
Category {Cat : "Ll" , Negate : negate },
Category {Cat : "Lu" , Negate : negate },
Category {Cat : "Lt" , Negate : negate })
}
c .addCategories (Category {Cat : categoryName , Negate : negate })
}
func (c *CharSet ) addCaseEquivalences () {
if c .anything {
return
}
rangeCount := len (c .ranges )
for i := 0 ; i < rangeCount ; i ++ {
r := c .ranges [i ]
for i := r .First ; i <= r .Last ; i ++ {
equiv := tryFindCaseEquivalences (i )
for _ , eq := range equiv {
c .ranges = append (c .ranges , SingleRange {First : eq , Last : eq })
}
}
}
c .canonicalize ()
}
func tryFindCaseEquivalences(ch rune ) []rune {
newCh := unicode .SimpleFold (ch )
if newCh == ch {
return nil
}
equiv := []rune {newCh }
for {
newCh = unicode .SimpleFold (newCh )
if newCh == ch {
return equiv
}
equiv = append (equiv , newCh )
}
}
func (c *CharSet ) addSubtraction (sub *CharSet ) {
c .sub = sub
}
func (c *CharSet ) addRange (chMin , chMax rune ) {
c .ranges = append (c .ranges , SingleRange {First : chMin , Last : chMax })
c .canonicalize ()
}
func (c *CharSet ) addNamedASCII (name string , negate bool ) bool {
var rs []SingleRange
switch name {
case "alnum" :
rs = []SingleRange {{'0' , '9' }, {'A' , 'Z' }, {'a' , 'z' }}
case "alpha" :
rs = []SingleRange {{'A' , 'Z' }, {'a' , 'z' }}
case "ascii" :
rs = []SingleRange {{0 , 0x7f }}
case "blank" :
rs = []SingleRange {{'\t' , '\t' }, {' ' , ' ' }}
case "cntrl" :
rs = []SingleRange {{0 , 0x1f }, {0x7f , 0x7f }}
case "digit" :
c .addDigit (false , negate )
case "graph" :
rs = []SingleRange {{'!' , '~' }}
case "lower" :
rs = []SingleRange {{'a' , 'z' }}
case "print" :
rs = []SingleRange {{' ' , '~' }}
case "punct" :
rs = []SingleRange {{'!' , '/' }, {':' , '@' }, {'[' , '`' }, {'{' , '~' }}
case "space" :
c .addSpace (true , false , negate )
case "upper" :
rs = []SingleRange {{'A' , 'Z' }}
case "word" :
c .addWord (true , negate )
case "xdigit" :
rs = []SingleRange {{'0' , '9' }, {'A' , 'F' }, {'a' , 'f' }}
default :
return false
}
if len (rs ) > 0 {
if negate {
c .addNegativeRanges (rs )
} else {
c .addRanges (rs )
}
}
return true
}
type singleRangeSorter []SingleRange
func (p singleRangeSorter ) Len () int { return len (p ) }
func (p singleRangeSorter ) Less (i , j int ) bool { return p [i ].First < p [j ].First }
func (p singleRangeSorter ) Swap (i , j int ) { p [i ], p [j ] = p [j ], p [i ] }
func (c *CharSet ) canonicalize () {
var i , j int
var last rune
if len (c .ranges ) == 0 {
return
}
if len (c .ranges ) > 1 {
sort .Sort (singleRangeSorter (c .ranges ))
done := false
for i , j = 1 , 0 ; ; i ++ {
for last = c .ranges [j ].Last ; ; i ++ {
if i == len (c .ranges ) || last >= unicode .MaxRune {
done = true
break
}
CurrentRange := c .ranges [i ]
if CurrentRange .First > last +1 {
break
}
if last < CurrentRange .Last {
last = CurrentRange .Last
}
}
c .ranges [j ] = SingleRange {First : c .ranges [j ].First , Last : last }
j ++
if done {
break
}
if j < i {
c .ranges [j ] = c .ranges [i ]
}
}
c .ranges = append (c .ranges [:j ], c .ranges [len (c .ranges ):]...)
}
if !c .negate && c .sub == nil && len (c .categories ) == 0 {
if len (c .ranges ) == 2 {
if c .ranges [0 ].First == 0 &&
c .ranges [1 ].Last >= unicode .MaxRune &&
c .ranges [0 ].Last < c .ranges [1 ].First -1 {
c .ranges = []SingleRange {{c .ranges [0 ].Last + 1 , c .ranges [1 ].First - 1 }}
c .negate = true
}
} else if len (c .ranges ) == 1 {
switch c .ranges [0 ].First {
case 0 :
if c .ranges [0 ].Last == unicode .MaxRune -1 {
c .ranges [0 ] = SingleRange {unicode .MaxRune , unicode .MaxRune }
c .negate = true
}
case 1 :
if c .ranges [0 ].Last >= unicode .MaxRune {
c .ranges [0 ] = SingleRange {'\x00' , '\x00' }
c .negate = true
}
}
}
}
if !c .negate &&
c .sub == nil &&
len (c .ranges ) == 1 && c .ranges [0 ].First == 0 && c .ranges [0 ].Last >= unicode .MaxRune {
c .makeAnything ()
}
if !c .negate && c .sub == nil && len (c .categories ) > 0 &&
len (c .ranges ) == 2 && c .ranges [0 ].First == 0 && c .ranges [0 ].Last +2 == c .ranges [1 ].First && c .ranges [1 ].Last == unicode .MaxRune {
if c .charInCategories (c .ranges [0 ].Last + 1 ) {
c .makeAnything ()
} else {
c .negate = true
c .ranges = []SingleRange {{c .ranges [0 ].Last + 1 , c .ranges [0 ].Last + 1 }}
c .categories = []Category {}
}
}
}
func (c *CharSet ) addLowercase () {
if c .anything {
return
}
toAdd := []SingleRange {}
for i := 0 ; i < len (c .ranges ); i ++ {
r := c .ranges [i ]
if r .First == r .Last {
lower := unicode .ToLower (r .First )
c .ranges [i ] = SingleRange {First : lower , Last : lower }
} else {
toAdd = append (toAdd , r )
}
}
for _ , r := range toAdd {
c .addLowercaseRange (r .First , r .Last )
}
c .canonicalize ()
}
const (
LowercaseSet = 0
LowercaseAdd = 1
LowercaseBor = 2
LowercaseBad = 3
)
type lcMap struct {
chMin, chMax rune
op, data int32
}
var lcTable = []lcMap {
{'\u0041' , '\u005A' , LowercaseAdd , 32 },
{'\u00C0' , '\u00DE' , LowercaseAdd , 32 },
{'\u0100' , '\u012E' , LowercaseBor , 0 },
{'\u0130' , '\u0130' , LowercaseSet , 0x0069 },
{'\u0132' , '\u0136' , LowercaseBor , 0 },
{'\u0139' , '\u0147' , LowercaseBad , 0 },
{'\u014A' , '\u0176' , LowercaseBor , 0 },
{'\u0178' , '\u0178' , LowercaseSet , 0x00FF },
{'\u0179' , '\u017D' , LowercaseBad , 0 },
{'\u0181' , '\u0181' , LowercaseSet , 0x0253 },
{'\u0182' , '\u0184' , LowercaseBor , 0 },
{'\u0186' , '\u0186' , LowercaseSet , 0x0254 },
{'\u0187' , '\u0187' , LowercaseSet , 0x0188 },
{'\u0189' , '\u018A' , LowercaseAdd , 205 },
{'\u018B' , '\u018B' , LowercaseSet , 0x018C },
{'\u018E' , '\u018E' , LowercaseSet , 0x01DD },
{'\u018F' , '\u018F' , LowercaseSet , 0x0259 },
{'\u0190' , '\u0190' , LowercaseSet , 0x025B },
{'\u0191' , '\u0191' , LowercaseSet , 0x0192 },
{'\u0193' , '\u0193' , LowercaseSet , 0x0260 },
{'\u0194' , '\u0194' , LowercaseSet , 0x0263 },
{'\u0196' , '\u0196' , LowercaseSet , 0x0269 },
{'\u0197' , '\u0197' , LowercaseSet , 0x0268 },
{'\u0198' , '\u0198' , LowercaseSet , 0x0199 },
{'\u019C' , '\u019C' , LowercaseSet , 0x026F },
{'\u019D' , '\u019D' , LowercaseSet , 0x0272 },
{'\u019F' , '\u019F' , LowercaseSet , 0x0275 },
{'\u01A0' , '\u01A4' , LowercaseBor , 0 },
{'\u01A7' , '\u01A7' , LowercaseSet , 0x01A8 },
{'\u01A9' , '\u01A9' , LowercaseSet , 0x0283 },
{'\u01AC' , '\u01AC' , LowercaseSet , 0x01AD },
{'\u01AE' , '\u01AE' , LowercaseSet , 0x0288 },
{'\u01AF' , '\u01AF' , LowercaseSet , 0x01B0 },
{'\u01B1' , '\u01B2' , LowercaseAdd , 217 },
{'\u01B3' , '\u01B5' , LowercaseBad , 0 },
{'\u01B7' , '\u01B7' , LowercaseSet , 0x0292 },
{'\u01B8' , '\u01B8' , LowercaseSet , 0x01B9 },
{'\u01BC' , '\u01BC' , LowercaseSet , 0x01BD },
{'\u01C4' , '\u01C5' , LowercaseSet , 0x01C6 },
{'\u01C7' , '\u01C8' , LowercaseSet , 0x01C9 },
{'\u01CA' , '\u01CB' , LowercaseSet , 0x01CC },
{'\u01CD' , '\u01DB' , LowercaseBad , 0 },
{'\u01DE' , '\u01EE' , LowercaseBor , 0 },
{'\u01F1' , '\u01F2' , LowercaseSet , 0x01F3 },
{'\u01F4' , '\u01F4' , LowercaseSet , 0x01F5 },
{'\u01FA' , '\u0216' , LowercaseBor , 0 },
{'\u0386' , '\u0386' , LowercaseSet , 0x03AC },
{'\u0388' , '\u038A' , LowercaseAdd , 37 },
{'\u038C' , '\u038C' , LowercaseSet , 0x03CC },
{'\u038E' , '\u038F' , LowercaseAdd , 63 },
{'\u0391' , '\u03AB' , LowercaseAdd , 32 },
{'\u03E2' , '\u03EE' , LowercaseBor , 0 },
{'\u0401' , '\u040F' , LowercaseAdd , 80 },
{'\u0410' , '\u042F' , LowercaseAdd , 32 },
{'\u0460' , '\u0480' , LowercaseBor , 0 },
{'\u0490' , '\u04BE' , LowercaseBor , 0 },
{'\u04C1' , '\u04C3' , LowercaseBad , 0 },
{'\u04C7' , '\u04C7' , LowercaseSet , 0x04C8 },
{'\u04CB' , '\u04CB' , LowercaseSet , 0x04CC },
{'\u04D0' , '\u04EA' , LowercaseBor , 0 },
{'\u04EE' , '\u04F4' , LowercaseBor , 0 },
{'\u04F8' , '\u04F8' , LowercaseSet , 0x04F9 },
{'\u0531' , '\u0556' , LowercaseAdd , 48 },
{'\u10A0' , '\u10C5' , LowercaseAdd , 48 },
{'\u1E00' , '\u1EF8' , LowercaseBor , 0 },
{'\u1F08' , '\u1F0F' , LowercaseAdd , -8 },
{'\u1F18' , '\u1F1F' , LowercaseAdd , -8 },
{'\u1F28' , '\u1F2F' , LowercaseAdd , -8 },
{'\u1F38' , '\u1F3F' , LowercaseAdd , -8 },
{'\u1F48' , '\u1F4D' , LowercaseAdd , -8 },
{'\u1F59' , '\u1F59' , LowercaseSet , 0x1F51 },
{'\u1F5B' , '\u1F5B' , LowercaseSet , 0x1F53 },
{'\u1F5D' , '\u1F5D' , LowercaseSet , 0x1F55 },
{'\u1F5F' , '\u1F5F' , LowercaseSet , 0x1F57 },
{'\u1F68' , '\u1F6F' , LowercaseAdd , -8 },
{'\u1F88' , '\u1F8F' , LowercaseAdd , -8 },
{'\u1F98' , '\u1F9F' , LowercaseAdd , -8 },
{'\u1FA8' , '\u1FAF' , LowercaseAdd , -8 },
{'\u1FB8' , '\u1FB9' , LowercaseAdd , -8 },
{'\u1FBA' , '\u1FBB' , LowercaseAdd , -74 },
{'\u1FBC' , '\u1FBC' , LowercaseSet , 0x1FB3 },
{'\u1FC8' , '\u1FCB' , LowercaseAdd , -86 },
{'\u1FCC' , '\u1FCC' , LowercaseSet , 0x1FC3 },
{'\u1FD8' , '\u1FD9' , LowercaseAdd , -8 },
{'\u1FDA' , '\u1FDB' , LowercaseAdd , -100 },
{'\u1FE8' , '\u1FE9' , LowercaseAdd , -8 },
{'\u1FEA' , '\u1FEB' , LowercaseAdd , -112 },
{'\u1FEC' , '\u1FEC' , LowercaseSet , 0x1FE5 },
{'\u1FF8' , '\u1FF9' , LowercaseAdd , -128 },
{'\u1FFA' , '\u1FFB' , LowercaseAdd , -126 },
{'\u1FFC' , '\u1FFC' , LowercaseSet , 0x1FF3 },
{'\u2160' , '\u216F' , LowercaseAdd , 16 },
{'\u24B6' , '\u24D0' , LowercaseAdd , 26 },
{'\uFF21' , '\uFF3A' , LowercaseAdd , 32 },
}
func (c *CharSet ) addLowercaseRange (chMin , chMax rune ) {
var i , iMax , iMid int
var chMinT , chMaxT rune
var lc lcMap
for i , iMax = 0 , len (lcTable ); i < iMax ; {
iMid = (i + iMax ) / 2
if lcTable [iMid ].chMax < chMin {
i = iMid + 1
} else {
iMax = iMid
}
}
for ; i < len (lcTable ); i ++ {
lc = lcTable [i ]
if lc .chMin > chMax {
return
}
chMinT = lc .chMin
if chMinT < chMin {
chMinT = chMin
}
chMaxT = lc .chMax
if chMaxT > chMax {
chMaxT = chMax
}
switch lc .op {
case LowercaseSet :
chMinT = rune (lc .data )
chMaxT = rune (lc .data )
case LowercaseAdd :
chMinT += lc .data
chMaxT += lc .data
case LowercaseBor :
chMinT |= 1
chMaxT |= 1
case LowercaseBad :
chMinT += (chMinT & 1 )
chMaxT += (chMaxT & 1 )
}
if chMinT < chMin || chMaxT > chMax {
c .ranges = append (c .ranges , SingleRange {First : chMinT , Last : chMaxT })
}
}
}
func (set1 *CharSet ) MayOverlap (set2 *CharSet ) bool {
if set1 .Equals (set2 ) {
return true
}
if set1 .IsAnything () || set2 .IsAnything () {
return true
}
set1Negated := set1 .IsNegated ()
set2Negated := set2 .IsNegated ()
if set1Negated != set2Negated {
return !set1 .equals (set2 , true )
}
if set1Negated {
return true
}
if knownDistinctSets (set1 , set2 ) || knownDistinctSets (set2 , set1 ) {
return false
}
if !set2 .HasSubtraction () && len (set2 .categories ) == 0 {
return mayOverlapByEnumeration (set1 , set2 )
} else if !set1 .HasSubtraction () && len (set1 .categories ) == 0 {
return mayOverlapByEnumeration (set2 , set1 )
}
return true
}
func knownDistinctSets(set1 , set2 *CharSet ) bool {
return (set1 .Equals (SpaceClass ()) || set1 .Equals (ECMASpaceClass ())) &&
(set2 .Equals (DigitClass ()) || set2 .Equals (WordClass ()) ||
set2 .Equals (ECMADigitClass ()) || set2 .Equals (ECMAWordClass ()))
}
func mayOverlapByEnumeration(set1 , set2 *CharSet ) bool {
for i := 0 ; i < len (set2 .ranges ); i ++ {
for c := set2 .ranges [i ].First ; c <= set2 .ranges [i ].Last ; c ++ {
if set1 .CharIn (c ) {
return true
}
}
}
return false
}
func (c *CharSet ) GetSetChars (maxChars int ) []rune {
if len (c .categories ) > 0 || len (c .ranges ) > maxChars {
return nil
}
if c .IsNegated () && c .HasSubtraction () {
return nil
}
chars := make ([]rune , 0 , maxChars )
curWork := 0
for _ , r := range c .ranges {
for ch := r .First ; ch <= r .Last ; ch ++ {
curWork ++
if curWork > maxChars {
return nil
}
if c .HasSubtraction () && !c .CharIn (ch ) {
continue
}
chars = append (chars , ch )
}
}
return chars
}
func (c *CharSet ) Hash () []byte {
b := &bytes .Buffer {}
c .mapHashFill (b )
return b .Bytes ()
}
func (c *CharSet ) Equals (c2 *CharSet ) bool {
return c .equals (c2 , false )
}
func (c *CharSet ) equals (c2 *CharSet , ignoreNegate bool ) bool {
if c == nil && c2 == nil {
return true
}
if c == nil && c2 != nil || c2 == nil && c != nil {
return false
}
if !ignoreNegate {
if c .negate != c2 .negate {
return false
}
}
if c .anything != c2 .anything {
return false
}
if !slices .Equal (c .ranges , c2 .ranges ) {
return false
}
if !slices .Equal (c .categories , c2 .categories ) {
return false
}
return c .sub .equals (c2 .sub , false )
}
var whitespaceChars = []rune {'\u0009' , '\u000A' , '\u000B' , '\u000C' , '\u000D' ,
'\u0020' , '\u0085' , '\u00A0' , '\u1680' , '\u2000' ,
'\u2001' , '\u2002' , '\u2003' , '\u2004' , '\u2005' ,
'\u2006' , '\u2007' , '\u2008' , '\u2009' , '\u200A' ,
'\u2028' , '\u2029' , '\u202F' , '\u205F' , '\u3000' }
func (c *CharSet ) IsUnicodeCategoryOfSmallCharCount () (isSmall bool , chars []rune , negated bool , desc string ) {
if c .IsSingleton () {
return true , []rune {c .SingletonChar ()}, false , ""
}
if c .IsSingletonInverse () {
return true , []rune {c .SingletonChar ()}, true , ""
}
if c .Equals (SpaceClass ()) {
return true , whitespaceChars , false , "whitespace"
}
if c .Equals (NotSpaceClass ()) {
return true , whitespaceChars , true , "whitespace"
}
return false , nil , false , ""
}
func (c *CharSet ) containsAsciiIgnoreCaseCharacter () (bool , []rune ) {
if c .IsNegated () {
return false , nil
}
twoChars := c .GetSetChars (3 )
return len (twoChars ) == 2 && twoChars [0 ] < unicode .MaxASCII && twoChars [1 ] < unicode .MaxASCII &&
(twoChars [0 ]|0x20 ) == (twoChars [1 ]|0x20 ) &&
unicode .IsLetter (twoChars [0 ]) && unicode .IsLetter (twoChars [1 ]), twoChars
}
func anyParticipateInCaseConversion(chars []rune ) bool {
for _ , c := range chars {
if participatesInCaseConversion (c ) {
return true
}
}
return false
}
func participatesInCaseConversion(ch rune ) bool {
return !unicode .In (ch , unicode .Pe , unicode .Pc , unicode .Cc , unicode .Pd , unicode .Nd , unicode .Pf ,
unicode .Pi , unicode .Zl , unicode .Ps , unicode .No , unicode .Po , unicode .Zp , unicode .Zs )
}
func anyParticipatesInCaseConversion(str string ) bool {
for _ , c := range str {
if participatesInCaseConversion (c ) {
return true
}
}
return false
}
func isAsciiRunes(in []rune ) bool {
for i := 0 ; i < len (in ); i ++ {
if in [i ] > unicode .MaxASCII {
return false
}
}
return true
}
func (c CharSet ) GetIfNRanges (n int ) []SingleRange {
if len (c .categories ) > 0 {
return nil
}
if c .sub != nil {
return nil
}
if len (c .ranges ) == n {
return c .ranges [:n ]
}
return nil
}
func (c *CharSet ) GetIfOnlyUnicodeCategories () (cats []Category , negate bool ) {
if c .sub != nil {
return nil , false
}
if len (c .ranges ) > 0 {
return nil , false
}
if len (c .categories ) == 0 {
return nil , false
}
neg := c .categories [0 ].Negate
for _ , cat := range c .categories {
if neg != cat .Negate || cat .Cat == SpaceCategoryText || cat .Cat == WordCategoryText {
return nil , false
}
}
return c .categories , neg != c .negate
}
type CharClassAnalysisResults struct {
OnlyRanges bool
ContainsOnlyAscii bool
ContainsNoAscii bool
AllAsciiContained bool
AllNonAsciiContained bool
LowerBoundInclusiveIfOnlyRanges rune
UpperBoundExclusiveIfOnlyRanges rune
}
func (set *CharSet ) Analyze () CharClassAnalysisResults {
if (set .IsNegated () && set .HasSubtraction ()) || len (set .categories ) > 0 || len (set .ranges ) == 0 {
return CharClassAnalysisResults {}
}
firstValueInclusive := set .ranges [0 ].First
lastValueExclusive := set .ranges [len (set .ranges )-1 ].Last + 1
if set .IsNegated () {
return CharClassAnalysisResults {
OnlyRanges : true ,
AllNonAsciiContained : lastValueExclusive <= unicode .MaxASCII ,
AllAsciiContained : firstValueInclusive >= unicode .MaxASCII ,
ContainsNoAscii : firstValueInclusive == 0 && set .ranges [0 ].Last >= unicode .MaxASCII ,
ContainsOnlyAscii : false ,
LowerBoundInclusiveIfOnlyRanges : firstValueInclusive ,
UpperBoundExclusiveIfOnlyRanges : lastValueExclusive ,
}
}
return CharClassAnalysisResults {
OnlyRanges : true ,
AllNonAsciiContained : false ,
AllAsciiContained : firstValueInclusive == 0 && set .ranges [0 ].Last >= unicode .MaxASCII && !set .HasSubtraction (),
ContainsOnlyAscii : lastValueExclusive <= unicode .MaxASCII ,
ContainsNoAscii : firstValueInclusive >= unicode .MaxASCII ,
LowerBoundInclusiveIfOnlyRanges : firstValueInclusive ,
UpperBoundExclusiveIfOnlyRanges : lastValueExclusive ,
}
}
The pages are generated with Golds v0.8.7 . (GOOS=linux GOARCH=amd64)
Golds is a Go 101 project developed by Tapir Liu .
PR and bug reports are welcome and can be submitted to the issue list .
Please follow @zigo_101 (reachable from the left QR code) to get the latest news of Golds .