package syntaximport ()// Computes a character class for the first character in tree. This uses a more robust algorithm// than is used by TryFindFixedLiterals and thus can find starting sets it couldn't. For example,// fixed literals won't find the starting set for a*b, as the a isn't guaranteed and the b is at a// variable position, but this will find [ab] as it's instead looking for anything that under any// circumstance could possibly start a match.func findFirstCharClass( *RegexNode) *CharSet {// Explore the graph, adding found chars into a result set, which is lazily initialized so that // we can initialize it to a parsed set if we discover one first (this is helpful not just for allocation // but because it enables supporting starting negated sets, which wouldn't work if they had to be merged // into a non-negated default set). If the operation returns true, we successfully explore all relevant nodes // in the graph. If it returns false, we were unable to successfully explore all relevant nodes, typically // due to conflicts when trying to add characters into the result set, e.g. we may have read a negated set // and were then unable to merge into that a subsequent non-negated set. If it returns null, it means the // whole pattern was nullable such that it could match an empty string, in which case we // can't make any statements about what begins a match.var *CharSetiftryFindFirstCharClass(, &) == 1 {return }returnnil}// Walks the nodes of the expression looking for any node that could possibly match the first// character of a match, e.g. in `a*b*c+d`, we'd find [abc], or in `(abc|d*e)?[fgh]`, we'd find// [adefgh]. The function is called for each node, recurring into children where appropriate,// and returns:// - 1 if the child was successfully processed and represents a stopping point, e.g. a single// char loop with a minimum bound greater than 0 such that nothing after that node in a// concatenation could possibly match the first character.// - 0 if the child failed to be processed but needed to be, such that the results can't// be trusted. If any node returns false, the whole operation fails.// - -1 if the child was successfully processed but doesn't represent a stopping point, i.e.// it's zero-width (e.g. empty, a lookaround, an anchor, etc.) or it could be zero-width// (e.g. a loop with a min bound of 0). A concatenation processing a child that returns// -1 needs to keep processing the next child.func tryFindFirstCharClass( *RegexNode, **CharSet) int { := *switch .T {// Base cases where we have results to add to the result set. Add the values into the result set, if possible. // If this is a loop and it has a lower bound of 0, then it's zero-width, so return null.caseNtOne, NtOneloop, NtOnelazy, NtOneloopatomic:if == nil { = &CharSet{} * = }if .IsMergeable() { .addChar(.Ch)if .T == NtOne || .M > 0 {return1 }return -1 }return0caseNtNotone, NtNotoneloop, NtNotonelazy, NtNotoneloopatomic:if == nil { = &CharSet{} * = }if .IsMergeable() { .addChar(.Ch) .negate = true/*if node.Ch > 0 { // Add the range before the excluded char. cc.addRange(0, (node.Ch - 1)) } if node.Ch < unicode.MaxRune { // Add the range after the excluded char. cc.addRange(node.Ch+1, unicode.MaxRune) }*/if .T == NtNotone || .M > 0 {return1 }return -1 }return0caseNtSet, NtSetloop, NtSetlazy, NtSetloopatomic: { := falseif == nil { := .Set.Copy() * = & = true } elseif .IsMergeable() && .Set.IsMergeable() { .addSet(*.Set) = true }if ! {return0 } elseif .T == NtSet || .M > 0 {return1 }return -1 }caseNtMulti:if == nil { = &CharSet{} * = }if .IsMergeable() {if .Options&RightToLeft != 0 { .addChar(.Str[len(.Str)-1]) } else { .addChar(.Str[0]) }return1 }return0caseNtGrapheme:// Every rune can begin a grapheme, so there is no useful candidate // restriction to derive.return0// Zero-width elements. These don't contribute to the starting set, so return null to indicate a caller // should keep looking past them.caseNtEmpty, NtNothing, NtBol, NtEol, NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary,NtBeginning, NtStart, NtEndZ, NtEnd, NtUpdateBumpalong, NtPosLook, NtNegLook:return -1// Groups. These don't contribute anything of their own, and are just pass-throughs to their children.caseNtAtomic, NtCapture:return (.Children[0], )// Loops. Like groups, these are mostly pass-through: if the child fails, then the whole operation needs // to fail, and if the child is nullable, then the loop is as well. However, if the child succeeds but // the loop has a lower bound of 0, then the loop is still nullable.caseNtLoop, NtLazyloop: := (.Children[0], )if <= 0 || .M != 0 {return }return -1// Concatenation. Loop through the children as long as they're nullable. The moment a child returns true, // we don't need or want to look further, as that child represents non-zero-width and nothing beyond it can // contribute to the starting character set. The moment a child returns false, we need to fail the whole thing. // If every child is nullable, then the concatenation is also nullable.caseNtConcatenate:for := 0; < len(.Children); ++ { := (.Children[], )if != -1 {return } }return -1// Alternation. Every child is its own fork/branch and contributes to the starting set. As with concatenation, // the moment any child fails, fail. And if any child is nullable, the alternation is also nullable (since that // zero-width path could be taken). Otherwise, if every branch returns true, so too does the alternation.caseNtAlternate: := falsefor := 0; < len(.Children); ++ { := (.Children[], )switch {case -1: = truecase0:return0 } }if {return -1 }return1// Conditionals. Just like alternation for their "yes"/"no" child branches. If either returns false, return false. // If either is nullable, this is nullable. If both return true, return true.caseNtBackRefCond, NtExprCond: := 1if .T == NtBackRefCond { = 0 }// conditional without all the branchesif +1 >= len(.Children) {return -1 } := (.Children[], ) := (.Children[+1], )if == -1 || == -1 {return -1 }if == 0 || == 0 {return0 }return1// Backreferences. We can't easily make any claims about what content they might match, so just give up.caseNtRef:return0 }// Unknown node.return0}// Computes the leading ordinal case-insensitive substring in nodefunc findPrefixOrdinalCaseInsensitive( *RegexNode) string {for {// Search down the left side of the tree looking for a concatenation. If we find one, // ask it for any ordinal case-insensitive prefix it has.switch .T {caseNtLoop, NtLazyloop:if .M <= 0 {return"" }fallthroughcaseNtAtomic, NtCapture: = .Children[0]continuecaseNtConcatenate: , , := .TryGetOrdinalCaseInsensitiveString(0, len(.Children), true)returndefault:return"" } }}// Computes the leading substring in node may be empty.func findPrefix( *RegexNode) string { := &bytes.Buffer{}tryFindPrefix(, )return .String()}// Processes the node, adding any prefix text to the builder.// Returns whether processing should continue with subsequent nodes.func tryFindPrefix( *RegexNode, *bytes.Buffer) bool {// We don't bother to handle reversed input, so process at most one node // when handling RightToLeft. := (.Options & RightToLeft) != 0switch .T {// ConcatenationcaseNtConcatenate:for := 0; < len(.Children); ++ {if !(.Children[], ) {returnfalse } }return !// Alternation: find a string that's a shared prefix of all branchescaseNtAlternate:// for RTL we'd need to be matching the suffixes of the alternation casesif {returnfalse }// Store the initial branch into the target builder, keeping track // of how much was appended. Any of this contents that doesn't overlap // will every other branch will be removed before returning. := .Len() (.Children[0], ) := .Len() - // Then explore the rest of the branches, finding the length // of prefix they all share in common with the initial branch.if != 0 { := &bytes.Buffer{} := .Bytes()[ : +]// Process each branch. If we reach a point where we've proven there's // no overlap, we can bail early.for := 1; < len(.Children) && != 0; ++ { .Reset()// Process the branch into a temporary builder. (.Children[], )// Find how much overlap there is between this branch's prefix // and the smallest amount of prefix that overlapped with all // the previously seen branches. = commonPrefixLen(, .Bytes()) }// Then cull back on what was added based on the other branches. .Truncate( + ) }// Don't explore anything after the alternation. We could make this work if desirable, // but it's currently not worth the extra complication. The entire contents of every // branch would need to be identical other than zero-width anchors/assertions.returnfalse// One charactercaseNtOne: .WriteRune(.Ch)return !// Multiple characterscaseNtMulti: .WriteString(string(.Str))return !// Loop of one charactercaseNtOneloop/*NtOneloopatomic,*/, NtOnelazy:if .M <= 0 {returnfalse }// arbitrary cut-off to avoid creating super long strings unnecessarily := 32if .M < { = .M } .WriteString(strings.Repeat(string(.Ch), ))return == .N && !// Loop of a nodecaseNtLoop, NtLazyloop:if .M <= 0 {returnfalse }// arbitrary cut-off to avoid creating super long strings unnecessarily := 4if .M < { = .M }for := 0; < ; ++ {if (.Children[0], ) {returnfalse } }return == .N && !// Grouping nodes for which we only care about their single childcaseNtAtomic, NtCapture:return (.Children[0], )// Zero-width anchors and assertionscaseNtBol, NtEol, NtBoundary, NtECMABoundary, NtNonboundary, NtNonECMABoundary, NtBeginning,NtStart, NtEndZ, NtEnd, NtEmpty, NtUpdateBumpalong, NtPosLook, NtNegLook:returntrue }// Give up for anything elsereturnfalse}// commonPrefixLen returns the length of the common prefix of two strings.func commonPrefixLen(, []byte) int { := min(len(), len()) := 0// Optimization: load and compare word-sized chunks at a time. // This is about 6x faster than the naive approach when len > 64. // // TODO(adonovan): further optimizations are possible, // at the cost of portability, for example by: // - better elimination of bounds checks; // - use of uint64 instead of an array may result in better // registerization, and allows computing the final portion // from the bitmask: // // cmp := load64le(a, i) ^ load64le(b, i) // if cmp != 0 { // return i + bits.LeadingZeros64(cmp)/8 // } // // - use of vector instructions in the manner of // runtime.cmpstring, which is expected to achieve 3x // further improvement when len > 32. //const = int(unsafe.Sizeof(uint(0)))var , []bytefor + <= {copy([:], [:+])copy([:], [:+])if != {break } += }// naive implementationfor < {if [] != [] {return } ++ }return}func min(, int) int {if < {return } else {return }}func max(, int) int {if > {return } else {return }}// Minimum string length for prefixes to be useful. If any prefix has length 1,// then we're generally better off just using IndexOfAny with chars.const minPrefixLength = 2// Arbitrary string length limit (with some wiggle room) to avoid creating strings that are longer than is useful and consuming too much memory.const maxPrefixLength = 8// Arbitrary limit on the number of prefixes to find. If we find more than this, we're likely to be spending too much time finding prefixes that won't be useful.const maxPrefixes = 16// Finds an array of multiple prefixes that a node can begin with.// If a fixed set of prefixes is found, such that a match for this node is guaranteed to begin// with one of those prefixes, an array of those prefixes is returned. Otherwise, null.func findPrefixes( *RegexNode, bool) []string {// Analyze the node to find prefixes. := []*bytes.Buffer{{}}findPrefixesCore(, &, )// If we found too many prefixes or if any found is too short, fail.iflen() > maxPrefixes || slices.ContainsFunc(, func( *bytes.Buffer) bool { return .Len() < minPrefixLength }) {returnnil }// Return the prefixes. := make([]string, len())for := 0; < len(); ++ { [] = [].String() }return}// Updates the results list with found prefixes. All existing strings in the list are treated as existing// discovered prefixes prior to the node being processed. The method returns true if subsequent nodes after// this one should be examined, or returns false if they shouldn't be because the node wasn't guaranteed// to be fully processed.func findPrefixesCore( *RegexNode, *[]*bytes.Buffer, bool) bool { := *// If we're too deep to analyze further, we can't trust what we've already computed, so stop iterating. // Also bail if any of our results is already hitting the threshold, or if this node is RTL, which is // not worth the complexity of handling. // Or if we've already discovered more than the allowed number of prefixes.ifslices.ContainsFunc(, func( *bytes.Buffer) bool { return .Len() >= maxPrefixLength }) || (.Options&RightToLeft) != 0 ||len() > maxPrefixes {returnfalse }// These limits are approximations. We'll stop trying to make strings longer once we exceed the max length, // and if we exceed the max number of prefixes by a non-trivial amount, we'll fail the operation. // limit how many chars we get from a set based on the max prefixes we care about //setChars := make([]rune, 0, maxPrefixes)// Loop down the left side of the tree, looking for a starting node we can handle. We only loop through // atomic and capture nodes, as the child is guaranteed to execute once, as well as loops with a positive // minimum and thus at least one guaranteed iteration.for {switch .T {// These nodes are all guaranteed to execute at least once, so we can just // skip through them to their child.caseNtAtomic, NtCapture: = .Children[0]continue// Zero-width anchors and assertions don't impact a prefix and may be skipped over.caseNtBol, NtEol, NtBoundary, NtECMABoundary, NtNonboundary, NtNonECMABoundary,NtBeginning, NtStart, NtEndZ, NtEnd, NtEmpty, NtUpdateBumpalong,NtPosLook, NtNegLook:returntrue// If we hit a single character, we can just return that character. // This is only relevant for case-sensitive searches, as for case-insensitive we'd have sets for anything // that produces a different result when case-folded, or for strings composed entirely of characters that // don't participate in case conversion. Single character loops are handled the same as single characters // up to the min iteration limit. We can continue processing after them as well if they're repeaters such // that their min and max are the same.caseNtOne, NtOneloop, NtOnelazy, NtOneloopatomic:if ! || !participatesInCaseConversion(.Ch) { := maxPrefixLengthif .T == NtOne { = 1 } elseif .M < { = .M }if == 1 {for , := range { .WriteRune(.Ch) } } else {for , := range { .WriteString(strings.Repeat(string(.Ch), )) } }return .T == NtOne || == .N }// If we hit a string, we can just return that string. // As with One above, this is only relevant for case-sensitive searches.caseNtMulti:if ! {for , := range { .WriteString(string(.Str)) } } else {// If we're ignoring case, then only append up through characters that don't participate in case conversion. // If there are any beyond that, we can't go further and need to stop with what we have.for , := range .Str {ifparticipatesInCaseConversion() {returnfalse }for , := range { .WriteRune() } } }returntrue// For case-sensitive, try to extract the characters that comprise it, and if there are // any and there aren't more than the max number of prefixes, we can return // them each as a prefix. Effectively, this is an alternation of the characters // that comprise the set. For case-insensitive, we need the set to be two ASCII letters that case fold to the same thing. // As with One and loops, set loops are handled the same as sets up to the min iteration limit.caseNtSet, NtSetloop, NtSetlazy, NtSetloopatomic: := .Set.GetSetChars(maxPrefixes)iflen() == 0 {returnfalse } := maxPrefixLengthif .T == NtSet { = 1 } elseif .M < { = .M }if ! {for := 0; < ; ++ { := len()if *len() > maxPrefixes {returnfalse }// Duplicate all of the existing strings for all of the new suffixes, other than the first.for , := range [1:] {for := 0; < ; ++ { := bytes.NewBuffer(slices.Clone([].Bytes())) .WriteRune() = append(, ) } } * = // Then append the first suffix to all of the existing strings.for := 0; < ; ++ { [].WriteRune([0]) } } } else {// For ignore-case, we currently only handle the simple (but common) case of a single // ASCII character that case folds to the same char. , := .Set.containsAsciiIgnoreCaseCharacter()if ! {returnfalse }// Append it to each.for , := range { .WriteString(strings.Repeat(string([1]), )) } } * = return .T == NtSet || == .NcaseNtConcatenate:for := 0; < len(.Children); ++ {if !(.Children[], , ) {returnfalse } }returntrue// We can append any guaranteed iterations as if they were a concatenation.caseNtLoop, NtLazyloop:if .M > 0 {// MaxPrefixLength here is somewhat arbitrary, as a single loop iteration could yield multiple chars := maxPrefixLengthif .M < { = .M }for := 0; < ; ++ {if !(.Children[0], , ) {returnfalse } }return == .N }// For alternations, we need to find a prefix for every branch; if we can't compute a // prefix for any one branch, we can't trust the results and need to give up, since we don't // know if our set of prefixes is complete.caseNtAlternate:// If there are more children than our maximum, just give up immediately, as we // won't be able to get a prefix for every branch and have it be within our max.iflen(.Children) > maxPrefixes {returnfalse }// Build up the list of all prefixes across all branches.var []*bytes.Buffer := []*bytes.Buffer{{}}for := 0; < len(.Children); ++ { (.Children[], &, )// If we now have too many results, bail.iflen()+len() > maxPrefixes {returnfalse }for , := range {// If a branch yields an empty prefix, then none of the other branches // matter, e.g. if the pattern is abc(def|ghi|), then this would result // in prefixes abcdef, abcghi, and abc, and since abc is a prefix of both // abcdef and abcghi, the former two would never be used.if .Len() == 0 {returnfalse } }if == nil { = = []*bytes.Buffer{{}} } else { = append(, ...) = []*bytes.Buffer{{}} } }// At this point, we know we can successfully incorporate the alternation's results // into the main results.// If the results are currently empty (meaning a single empty StringBuilder), we can remove // that builder and just replace the results with the alternation's results. We would otherwise // be creating a dot product of every builder in the results with every branch's result, which // is logically the same thing.iflen() == 1 && [0].Len() == 0 { = } else { := len()// Duplicate all of the existing strings for all of the new suffixes, other than the first.for := 1; < len(); ++ { := []for := 0; < ; ++ { := &bytes.Buffer{} .Write([].Bytes()) .Write(.Bytes()) = append(, ) } }// Then append the first suffix to all of the existing strings.for := 0; < ; ++ { [].Write([0].Bytes()) } } * = // We don't know that we fully processed every branch, so we can't iterate through what comes after this node. // The results were successfully updated, but return false to indicate that nothing after this node should be examined. }returnfalse }}const maxLoopExpansion = 20// arbitrary cut-off to avoid loops adding significant overhead to processingconst maxFixedResults = 50// arbitrary cut-off to avoid generating lots of sets unnecessarily// Finds sets at fixed-offsets from the beginning of the pattern/</summary>// set "thorough" to true to spend more time finding sets (e.g. through alternations); false to do a faster analysis that's potentially more incomplete.// Returns the array of found sets, or null if there aren't any.func findFixedDistanceSets( *RegexNode, bool) []FixedDistanceSet {// Find all fixed-distance sets. := []FixedDistanceSet{} := 0tryFindRawFixedSets(, &, &, )// Remove any sets that match everything; they're not helpful. (This check exists primarily to weed // out use of . in Singleline mode, but also filters out explicit sets like [\s\S].) = slices.DeleteFunc(, func( FixedDistanceSet) bool { return .Set.IsAnything() })// If we don't have any results, try harder to compute one for the starting character. // This is a more involved computation that can find things the fixed-distance investigation // doesn't.iflen() == 0 {// weed out match-all, same as above := findFirstCharClass()if == nil || .IsAnything() {returnnil } = append(, FixedDistanceSet{Set: , Distance: 0}) }// For every entry, try to get the chars that make up the set, if there are few enough. // For any for which we couldn't get the small chars list, see if we can get other useful info. //scratch := make([]rune, 0, 128)for := 0; < len(); ++ { := [] .Negated = .Set.IsNegated()// prefer IndexOfAny for tiny sets of 1 or 2 elementsif := .Set.GetIfNRanges(1); len() == 1 && [0].Last-[0].First > 1 { .Range = &[0] } else { := .Set.GetSetChars(128)iflen() > 0 { .Chars = } } [] = }return}// Starting from the specified root node, populates results with any characters at a fixed distance// from the node's starting position. The function returns true if the entire contents of the node// is at a fixed distance, in which case distance will have been updated to include the full length// of the node. If it returns false, the node isn't entirely fixed, in which case subsequent nodes// shouldn't be examined and distance should no longer be trusted. However, regardless of whether it// returns true or false, it may have populated results, and all populated results are valid. All// FixedDistanceSet result will only have its Set string and Distance populated; the rest is left// to be populated by FindFixedDistanceSets after this returns.func tryFindRawFixedSets( *RegexNode, *[]FixedDistanceSet, *int, bool) bool { := *if .Options&RightToLeft != 0 {returnfalse }switch .T {caseNtOne:iflen() < maxFixedResults { := &CharSet{} .addChar(.Ch) = append(, FixedDistanceSet{Set: , Distance: *}) * = *++returntrue }returnfalsecaseNtOnelazy, NtOneloop, NtOneloopatomic:if .M > 0 { := &CharSet{} .addChar(.Ch) := maxLoopExpansionif .M < { = .M } := 0for ; < && len() < maxFixedResults; ++ { = append(, FixedDistanceSet{Set: , Distance: *}) *++ } * = return == .M && == .N }caseNtMulti: := 0for ; < len(.Str) && len() < maxFixedResults; ++ { := &CharSet{} .addChar(.Str[]) = append(, FixedDistanceSet{Set: , Distance: *}) *++ } * = return == len(.Str)caseNtSet:iflen() < maxFixedResults { = append(, FixedDistanceSet{Set: .Set, Distance: *}) * = *++returntrue }returnfalsecaseNtSetlazy, NtSetloop, NtSetloopatomic:if .M > 0 { := maxLoopExpansionif .M < { = .M } := 0for ; < && len() < maxFixedResults; ++ { = append(, FixedDistanceSet{Set: .Set, Distance: *}) *++ } * = return == .M && == .N }caseNtNotone:// We could create a set out of Notone, but it will be of little value in helping to improve // the speed of finding the first place to match, as almost every character will match it. *++returntruecaseNtNotonelazy, NtNotoneloop, NtNotoneloopatomic:if .M == .N { * += .Mreturntrue }caseNtBeginning, NtBol, NtBoundary, NtECMABoundary, NtEmpty, NtEnd, NtEndZ, NtEol,NtNonboundary, NtNonECMABoundary, NtUpdateBumpalong,NtStart, NtNegLook, NtPosLook:// Zero-width anchors and assertions. In theory, for PositiveLookaround and NegativeLookaround we could also // investigate them and use the learned knowledge to impact the generated sets, at least for lookaheads. // For now, we don't bother.returntruecaseNtAtomic, NtGroup, NtCapture:return (.Children[0], , , )caseNtLazyloop, NtLoop:if .M > 0 {// This effectively only iterates the loop once. If deemed valuable, // it could be updated in the future to duplicate the found results // (updated to incorporate distance from previous iterations) and // summed distance for all node.M iterations. If node.M == node.N, // this would then also allow continued evaluation of the rest of the // expression after the loop. (.Children[0], , , )returnfalse }caseNtConcatenate:for := 0; < len(.Children); ++ {if !(.Children[], , , ) {returnfalse } }returntruecaseNtAlternate:if { := true := -1var = make(map[int]struct { *CharSetint })var []FixedDistanceSetfor := 0; < len(.Children); ++ { = []FixedDistanceSet{} := 0 = && (.Children[], &, &, )iflen() == 0 {returnfalse }if {if == -1 { = } elseif != { = false } }for , := range {if , := [.Distance]; {if ..IsMergeable() && .Set.IsMergeable() { ..addSet(*.Set) .++ [.Distance] = } } else { := .Set.Copy() [.Distance] = struct { *CharSetint }{: &, : 1} } } }for , := range {iflen() >= maxFixedResults { = falsebreak }if . == len(.Children) { = append(, FixedDistanceSet{Set: ., Distance: + *}) } } * = if { * += returntrue }returnfalse } }returnfalse}// CompareFunc for a set of fixed-distance set results from best to worst quality.</summary>func compareFixedDistanceSetsByQuality(, FixedDistanceSet) int {// Finally, try to move the "best" results to be earlier. "best" here are ones we're able to search // for the fastest and that have the best chance of matching as few false positives as possible. := getRangeLength(.Range, .Negated) := getRangeLength(.Range, .Negated)// If one set is negated and the other isn't, prefer the non-negated set. In general, negated // sets are large and thus likely to match more frequently, making them slower to search for.if .Negated != .Negated {if .Negated {return -1 }return1 }// If we extracted only a few chars and the sets are negated, they both represent very large // sets that are difficult to compare for quality.if !.Negated {// If both have chars, prioritize the one with the smaller frequency for those chars.iflen(.Chars) > 0 && len(.Chars) > 0 {// Prefer sets with less frequent values. The frequency is only an approximation, // used as a tie-breaker when we'd otherwise effectively be picking randomly. // True frequencies will vary widely based on the actual data being searched, the language of the data, etc. := sumFrequencies(.Chars) := sumFrequencies(.Chars)if != {returncmp.Compare(, ) }if !isAsciiRunes(.Chars) && !isAsciiRunes(.Chars) {// Prefer the set with fewer values.returncmp.Compare(len(.Chars), len(.Chars)) } }// If one has chars and the other has a range, prefer the shorter set.if (len(.Chars) > 0 && > 0) || ( > 0 && len(.Chars) > 0) { := iflen(.Chars) > { = len(.Chars) } := iflen(.Chars) > { = len(.Chars) } := cmp.Compare(, )if != 0 {return }// If lengths are the same, prefer the chars.iflen(.Chars) > 0 {return -1 }return1 }// If one has chars and the other doesn't, prioritize the one with chars.if (len(.Chars) > 0) != (len(.Chars) > 0) {iflen(.Chars) > 0 {return -1 }return1 } }// If one has a range and the other doesn't, prioritize the one with a range.if ( > 0) != ( > 0) {if > 0 {return -1 }return1 }// If both have ranges, prefer the one that includes fewer characters.if > 0 {returncmp.Compare(, ) }// As a tiebreaker, prioritize the earlier one.returncmp.Compare(.Distance, .Distance)}func getRangeLength( *SingleRange, bool) int {if == nil {return0 }if {returnint(unicode.MaxRune - (.Last - .First)) }returnint(.Last - .First + 1)}func sumFrequencies( []rune) float32 {varfloat32for , := range {// Lookup each character in the table. Values >= 128 are ignored // and thus we'll get skew in the data. It's already a gross approximation, though, // and it is primarily meant for disambiguation of ASCII letters.if < unicode.MaxASCII { += frequency[] } }return}func hasHighFrequencyChars( FixedDistanceSet) bool {if .Negated {returntrue }// Sets without extracted chars can't be frequency-analyzed. // Single-char sets use IndexOf, which is a strong filter regardless of frequency.iflen(.Chars) <= 1 {returnfalse } := sumFrequencies(.Chars)// If the average frequency of the set's chars exceeds this threshold, the // characters are common enough that a multi-string search may be a better filter.const = 0.6return >= *float32(len(.Chars))}func mayContainCaseInsensitiveMatching( *RegexNode) bool {if .Options&IgnoreCase != 0 {returntrue }if .Set != nil { := .Set.GetSetChars(maxPrefixes)for , := range {ifparticipatesInCaseConversion() &&slices.Contains(, unicode.ToLower()) &&slices.Contains(, unicode.ToUpper()) {returntrue } } }for , := range .Children {if () {returntrue } }returnfalse}// Percent occurrences in source text (100 * char count / total count)var frequency = []float32{0.000/* '\x00' */, 0.000/* '\x01' */, 0.000/* '\x02' */, 0.000/* '\x03' */, 0.000/* '\x04' */, 0.000/* '\x05' */, 0.000/* '\x06' */, 0.000, /* '\x07' */0.000/* '\x08' */, 0.001/* '\x09' */, 0.000/* '\x0A' */, 0.000/* '\x0B' */, 0.000/* '\x0C' */, 0.000/* '\x0D' */, 0.000/* '\x0E' */, 0.000, /* '\x0F' */0.000/* '\x10' */, 0.000/* '\x11' */, 0.000/* '\x12' */, 0.000/* '\x13' */, 0.003/* '\x14' */, 0.000/* '\x15' */, 0.000/* '\x16' */, 0.000, /* '\x17' */0.000/* '\x18' */, 0.004/* '\x19' */, 0.000/* '\x1A' */, 0.000/* '\x1B' */, 0.006/* '\x1C' */, 0.006/* '\x1D' */, 0.000/* '\x1E' */, 0.000, /* '\x1F' */8.952/* ' ' */, 0.065/* ' !' */, 0.420/* ' "' */, 0.010/* ' #' */, 0.011/* ' $' */, 0.005/* ' %' */, 0.070/* ' &' */, 0.050, /* ' '' */3.911/* ' (' */, 3.910/* ' )' */, 0.356/* ' *' */, 2.775/* ' +' */, 1.411/* ' ,' */, 0.173/* ' -' */, 2.054/* ' .' */, 0.677, /* ' /' */1.199/* ' 0' */, 0.870/* ' 1' */, 0.729/* ' 2' */, 0.491/* ' 3' */, 0.335/* ' 4' */, 0.269/* ' 5' */, 0.435/* ' 6' */, 0.240, /* ' 7' */0.234/* ' 8' */, 0.196/* ' 9' */, 0.144/* ' :' */, 0.983/* ' ;' */, 0.357/* ' <' */, 0.661/* ' =' */, 0.371/* ' >' */, 0.088, /* ' ?' */0.007/* ' @' */, 0.763/* ' A' */, 0.229/* ' B' */, 0.551/* ' C' */, 0.306/* ' D' */, 0.449/* ' E' */, 0.337/* ' F' */, 0.162, /* ' G' */0.131/* ' H' */, 0.489/* ' I' */, 0.031/* ' J' */, 0.035/* ' K' */, 0.301/* ' L' */, 0.205/* ' M' */, 0.253/* ' N' */, 0.228, /* ' O' */0.288/* ' P' */, 0.034/* ' Q' */, 0.380/* ' R' */, 0.730/* ' S' */, 0.675/* ' T' */, 0.265/* ' U' */, 0.309/* ' V' */, 0.137, /* ' W' */0.084/* ' X' */, 0.023/* ' Y' */, 0.023/* ' Z' */, 0.591/* ' [' */, 0.085/* ' \' */, 0.590/* ' ]' */, 0.013/* ' ^' */, 0.797, /* ' _' */0.001/* ' `' */, 4.596/* ' a' */, 1.296/* ' b' */, 2.081/* ' c' */, 2.005/* ' d' */, 6.903/* ' e' */, 1.494/* ' f' */, 1.019, /* ' g' */1.024/* ' h' */, 3.750/* ' i' */, 0.286/* ' j' */, 0.439/* ' k' */, 2.913/* ' l' */, 1.459/* ' m' */, 3.908/* ' n' */, 3.230, /* ' o' */1.444/* ' p' */, 0.231/* ' q' */, 4.220/* ' r' */, 3.924/* ' s' */, 5.312/* ' t' */, 2.112/* ' u' */, 0.737/* ' v' */, 0.573, /* ' w' */0.992/* ' x' */, 1.067/* ' y' */, 0.181/* ' z' */, 0.391/* ' {' */, 0.056/* ' |' */, 0.391/* ' }' */, 0.002/* ' ~' */, 0.000, /* '\x7F' */}// The above table was generated programmatically with the following. This can be augmented to incorporate additional data sources,// though it is only intended to be a rough approximation use when tie-breaking and we'd otherwise be picking randomly, so, it's something.// The frequencies may be wildly inaccurate when used with data sources different in nature than the training set, in which case we shouldn't// be much worse off than just picking randomly://// using System.Runtime.InteropServices;//// var counts = new Dictionary<byte, long>();//// (string, string)[] rootsAndExtensions = new[]// {// (@"d:\repos\runtime\src\", "*.cs"), // C# files in dotnet/runtime// (@"d:\Top25GutenbergBooks", "*.txt"), // Top 25 most popular books on Project Gutenberg// };//// foreach ((string root, string ext) in rootsAndExtensions)// foreach (string path in Directory.EnumerateFiles(root, ext, SearchOption.AllDirectories))// foreach (string line in File.ReadLines(path))// foreach (char c in line.AsSpan().Trim())// CollectionsMarshal.GetValueRefOrAddDefault(counts, (byte)c, out _)++;//// long total = counts.Sum(i => i.Value);//// Console.WriteLine("/// <summary>Percent occurrences in source text (100 * char count / total count).</summary>");// Console.WriteLine("private static ReadOnlySpan<float> Frequency =>");// Console.WriteLine("[");// int i = 0;// for (int row = 0; row < 16; row++)// {// Console.Write(" ");// for (int col = 0; col < 8; col++)// {// counts.TryGetValue((byte)i, out long charCount);// float frequency = (float)(charCount / (double)total) * 100;// Console.Write($" {frequency:N3}f /* '{(i >= 32 && i < 127 ? $" {(char)i}" : $"\\x{i:X2}")}' */,");// i++;// }// Console.WriteLine();// }// Console.WriteLine("];");// / <summary>// / Analyzes the pattern for a leading set loop followed by a non-overlapping literal. If such a pattern is found, an implementation// / can search for the literal and then walk backward through all matches for the loop until the beginning is found.// / </summary>func findLiteralFollowingLeadingLoop( *RegexNode) *LiteralAfterLoop {if (.Options & RightToLeft) != 0 {// As a simplification, ignore RightToLeft.returnnil }// Find the first concatenation. We traverse through atomic and capture nodes as they don't effect flow control. (We don't // want to explore loops, even if they have a guaranteed iteration, because we may use information about the node to then // skip the node's execution in the matching algorithm, and we would need to special-case only skipping the first iteration.)for .T == NtAtomic || .T == NtCapture { = .Children[0] }if .T != NtConcatenate {returnnil }// Bail if the first node isn't a set loop. We treat any kind of set loop (Setloop, Setloopatomic, and Setlazy) // the same because of two important constraints: the loop must not have an upper bound, and the literal we look // for immediately following it must not overlap. With those constraints, all three of these kinds of loops will // end up having the same semantics; in fact, if atomic optimizations are used, we will have converted Setloop // into a Setloopatomic (but those optimizations are disabled for NonBacktracking in general). This // could also be made to support Oneloopatomic and Notoneloopatomic, but the scenarios for that are rare. := .Children[0]for .T == NtAtomic || .T == NtCapture { = .Children[0] }if (.T != NtSetloop && .T != NtSetloopatomic && .T != NtSetlazy) || .N != math.MaxInt32 {returnnil }// Get the subsequent node. An UpdateBumpalong may have been added as an optimization, but it doesn't have an // impact on semantics and we can skip it. := .Children[1]if .T == NtUpdateBumpalong {iflen(.Children) == 2 {// If the UpdateBumpalong is the last node, nothing meaningful follows the set loop.returnnil } = .Children[2] } = unwrapImmediateLiteralAfterLoopNode()if == nil {returnnil }// Is the set loop followed by a case-sensitive string we can search for?if := findPrefix(); len() >= 1 {// The literal can be searched for as either a single char or as a string. // But we need to make sure that its starting character isn't part of the preceding // set, as then we can't know for certain where the set loop ends.if .Set.CharIn(rune([0])) {returnnil } elseiflen() == 1 {return &LiteralAfterLoop{LoopNode: ,Char: rune([0]), } }return &LiteralAfterLoop{LoopNode: ,String: , } }// Is the set loop followed by an ordinal case-insensitive string we can search for? We could // search for a string with at least one char, but if it has only one, we're better off just // searching as a set, so we look for strings with at least two chars.if := findPrefixOrdinalCaseInsensitive(); len() >= 2 {// The literal can be searched for as a case-insensitive string. As with ordinal above, // though, we need to make sure its starting character isn't part of the previous set. // If that starting character participates in case conversion, then we need to test out // both casings (FindPrefixOrdinalCaseInsensitive will only return strings composed of // characters that either are ASCII or that don't participate in case conversion). , := utf8.DecodeRuneInString()ifparticipatesInCaseConversion() {if .Set.CharIn(|0x20) || .Set.CharIn(&^0x20) {returnnil } } elseif .Set.CharIn() {returnnil }return &LiteralAfterLoop{LoopNode: ,String: ,StringIgnoreCase: true, } }// If the resulting node is a set with at least one iteration, we can search for it.if .IsSetFamily() && !.Set.IsNegated() && (.T == NtSet || .M >= 1) {// maximum number of chars optimized by IndexOfAny := .Set.GetSetChars(5)iflen() > 0 {for , := range {if .Set.CharIn() {returnnil } }return &LiteralAfterLoop{LoopNode: ,Chars: , } } }// Otherwise, we couldn't find the pattern of an atomic set loop followed by a literal.returnnil}func unwrapImmediateLiteralAfterLoopNode( *RegexNode) *RegexNode {for { = unwrapTransparentNodes()if == nil {returnnil }if .T != NtConcatenate {return }iflen(.Children) == 0 {returnnil } = .Children[0] }}func findRequiredLandmarkChain( *RegexNode) *RequiredLandmarkChain {if (.Options & RightToLeft) != 0 {returnnil } = unwrapTransparentNodes()if .T != NtConcatenate || len(.Children) < 4 {returnnil } := unwrapTransparentNodes(.Children[0])if !isUnboundedSetLoop() {returnnil }// Collect landmarks that must appear in order later in the match. This is a // conservative prefilter: if any child cannot be described as a detectable // landmark, we skip that child rather than making it part of the chain. := make([]RequiredLandmark, 0, 2)for , := range .Children[1:] {if , := extractRequiredLandmark(); { = append(, ) } elseiflen() == 0 && !isZeroWidthLandmarkGap() {returnnil } }iflen() < 2 {returnnil }return &RequiredLandmarkChain{LeadingLoopSet: .Set,Landmarks: , }}func isZeroWidthLandmarkGap( *RegexNode) bool { = unwrapTransparentNodes()if == nil {returntrue }switch .T {caseNtEmpty, NtUpdateBumpalong,NtBeginning, NtBol, NtStart, NtEndZ, NtEnd, NtEol,NtBoundary, NtNonboundary, NtECMABoundary, NtNonECMABoundary:returntruedefault:returnfalse }}func unwrapTransparentNodes( *RegexNode) *RegexNode {for != nil && (.T == NtAtomic || .T == NtCapture || .T == NtGroup) && len(.Children) == 1 { = .Children[0] }return}func isUnboundedSetLoop( *RegexNode) bool {return != nil && (.T == NtSetloop || .T == NtSetloopatomic || .T == NtSetlazy) && .Set != nil && .N == math.MaxInt32}func extractRequiredLandmark( *RegexNode) (RequiredLandmark, bool) { = unwrapTransparentNodes()if == nil {returnRequiredLandmark{}, false }if .T != NtAlternate { , := extractRequiredLandmarkAlternative()if ! {returnRequiredLandmark{}, false }returnRequiredLandmark{Alternatives: []RequiredLandmarkAlternative{}}, true } := RequiredLandmark{Alternatives: make([]RequiredLandmarkAlternative, 0, len(.Children))}for , := range .Children { , := extractRequiredLandmarkAlternative()if ! {returnRequiredLandmark{}, false } .Alternatives = append(.Alternatives, ) }return , len(.Alternatives) > 0}func extractRequiredLandmarkAlternative( *RegexNode) (RequiredLandmarkAlternative, bool) { = unwrapTransparentNodes()if == nil {returnRequiredLandmarkAlternative{}, false } := []*RegexNode{}if .T == NtConcatenate { = .Children } := RequiredLandmarkAlternative{} := 0if < len() {if , , := whitespaceLoop([]); { .LeadingWhitespaceSet = .RequireWhitespaceBefore = > 0 ++ } }if >= len() {returnRequiredLandmarkAlternative{}, false }// The core of a landmark must be a literal or a bounded, enumerable set // repetition. The VM still validates the full regex after this prefilter // returns a candidate. := unwrapTransparentNodes([])switch .T {caseNtOne: .Literal = []rune{.Ch} .MinRepeat = 1 .MaxRepeat = 1caseNtMulti: .Literal = .Str .MinRepeat = 1 .MaxRepeat = 1caseNtSet, NtSetloop, NtSetloopatomic, NtSetlazy:if .Set == nil {returnRequiredLandmarkAlternative{}, false }if .T == NtSet { .MinRepeat = 1 .MaxRepeat = 1 } elseif .M > 0 && .N != math.MaxInt32 { .MinRepeat = .M .MaxRepeat = .N } else {returnRequiredLandmarkAlternative{}, false } := .Set.GetSetChars(8)iflen() == 0 || .Set.IsNegated() {returnRequiredLandmarkAlternative{}, false } .Set = .Setdefault:returnRequiredLandmarkAlternative{}, false } ++if < len() {if , , := whitespaceLoop([]); { .TrailingWhitespaceSet = .RequireWhitespaceAfter = > 0 ++ } }if != len() {returnRequiredLandmarkAlternative{}, false }return , true}func whitespaceLoop( *RegexNode) (*CharSet, int, bool) { = unwrapTransparentNodes()if != nil && (.T == NtSetloop || .T == NtSetloopatomic || .T == NtSetlazy) && .Set != nil && .N == math.MaxInt32 && (.Set.Equals(SpaceClass()) || .Set.Equals(ECMASpaceClass()) || .Set.Equals(RE2SpaceClass())) {return .Set, .M, true }returnnil, 0, false}// Returns a leading positive lookahead if found and whether to keep examining subsequent nodes in a concatenation.func findLeadingPositiveLookahead( *RegexNode) (*RegexNode, bool) {for {if .Options&RightToLeft != 0 {returnnil, false }switch .T {caseNtPosLook:return , falsecaseNtBol, NtEol, NtBeginning, NtStart, NtEndZ, NtEnd,NtBoundary, NtECMABoundary, NtNegLook, NtEmpty:returnnil, truecaseNtAtomic, NtCapture: = .Children[0]continuecaseNtLoop, NtLazyloop:if .M < 1 {returnnil, false } , := (.Children[0])return , falsecaseNtConcatenate:for := 0; < len(.Children); ++ { , := (.Children[])if != nil || ! {return , false } }returnnil, truedefault:returnnil, false } }}
The pages are generated with Goldsv0.8.7. (GOOS=linux GOARCH=amd64)
Golds is a Go 101 project developed by Tapir Liu.
PR and bug reports are welcome and can be submitted to the issue list.
Please follow @zigo_101 (reachable from the left QR code) to get the latest news of Golds.