package regalloc
import (
"fmt"
"math"
"strings"
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
)
func NewAllocator [I Instr , B Block [I ], F Function [I , B ]](allocatableRegs *RegisterInfo ) Allocator [I , B , F ] {
a := Allocator [I , B , F ]{
regInfo : allocatableRegs ,
phiDefInstListPool : wazevoapi .NewPool [phiDefInstList [I ]](resetPhiDefInstList [I ]),
blockStates : wazevoapi .NewIDedPool [blockState [I , B , F ]](resetBlockState [I , B , F ]),
}
a .state .vrStates = wazevoapi .NewIDedPool [vrState [I , B , F ]](resetVrState [I , B , F ])
a .state .reset ()
for _ , regs := range allocatableRegs .AllocatableRegisters {
for _ , r := range regs {
a .allocatableSet = a .allocatableSet .add (r )
}
}
return a
}
type (
RegisterInfo struct {
AllocatableRegisters [NumRegType ][]RealReg
CalleeSavedRegisters RegSet
CallerSavedRegisters RegSet
RealRegToVReg []VReg
RealRegName func (r RealReg ) string
RealRegType func (r RealReg ) RegType
}
Allocator [I Instr , B Block [I ], F Function [I , B ]] struct {
regInfo *RegisterInfo
allocatableSet RegSet
allocatedCalleeSavedRegs []VReg
vs []VReg
ss []*vrState [I , B , F ]
copies []_copy [I , B , F ]
phiDefInstListPool wazevoapi .Pool [phiDefInstList [I ]]
blks []B
reals []RealReg
state state [I , B , F ]
blockStates wazevoapi .IDedPool [blockState [I , B , F ]]
}
_copy[I Instr , B Block [I ], F Function [I , B ]] struct {
src *vrState [I , B , F ]
dstID VRegID
}
programCounter int32
state[I Instr , B Block [I ], F Function [I , B ]] struct {
argRealRegs []VReg
regsInUse regInUseSet [I , B , F ]
vrStates wazevoapi .IDedPool [vrState [I , B , F ]]
currentBlockID int32
allocatedRegSet RegSet
}
blockState[I Instr , B Block [I ], F Function [I , B ]] struct {
liveIns []*vrState [I , B , F ]
seen bool
visited bool
startFromPredIndex int
startRegs regInUseSet [I , B , F ]
endRegs regInUseSet [I , B , F ]
}
vrState[I Instr , B Block [I ], f Function [I , B ]] struct {
v VReg
r RealReg
defInstr I
defBlk B
lca B
lastUse programCounter
lastUseUpdatedAtBlockID int32
spilled bool
isPhi bool
desiredLoc desiredLoc
*phiDefInstList [I ]
}
phiDefInstList[I Instr ] struct {
instr I
v VReg
next *phiDefInstList [I ]
}
desiredLoc uint16
desiredLocKind uint16
)
const (
desiredLocKindUnspecified desiredLocKind = iota
desiredLocKindStack
desiredLocKindReg
desiredLocUnspecified = desiredLoc (desiredLocKindUnspecified )
desiredLocStack = desiredLoc (desiredLocKindStack )
)
func newDesiredLocReg(r RealReg ) desiredLoc {
return desiredLoc (desiredLocKindReg ) | desiredLoc (r <<2 )
}
func (d desiredLoc ) realReg () RealReg {
return RealReg (d >> 2 )
}
func (d desiredLoc ) stack () bool {
return d &3 == desiredLoc (desiredLocKindStack )
}
func resetPhiDefInstList[I Instr ](l *phiDefInstList [I ]) {
var nilInstr I
l .instr = nilInstr
l .next = nil
l .v = VRegInvalid
}
func (s *state [I , B , F ]) dump (info *RegisterInfo ) {
fmt .Println ("\t\tstate:" )
fmt .Println ("\t\t\targRealRegs:" , s .argRealRegs )
fmt .Println ("\t\t\tregsInUse" , s .regsInUse .format (info ))
fmt .Println ("\t\t\tallocatedRegSet:" , s .allocatedRegSet .format (info ))
fmt .Println ("\t\t\tused:" , s .regsInUse .format (info ))
var strs []string
for i := 0 ; i <= s .vrStates .MaxIDEncountered (); i ++ {
vs := s .vrStates .Get (i )
if vs == nil {
continue
}
if vs .r != RealRegInvalid {
strs = append (strs , fmt .Sprintf ("(v%d: %s)" , vs .v .ID (), info .RealRegName (vs .r )))
}
}
fmt .Println ("\t\t\tvrStates:" , strings .Join (strs , ", " ))
}
func (s *state [I , B , F ]) reset () {
s .argRealRegs = s .argRealRegs [:0 ]
s .vrStates .Reset ()
s .allocatedRegSet = RegSet (0 )
s .regsInUse .reset ()
s .currentBlockID = -1
}
func resetVrState[I Instr , B Block [I ], F Function [I , B ]](vs *vrState [I , B , F ]) {
vs .v = VRegInvalid
vs .r = RealRegInvalid
var nilInstr I
vs .defInstr = nilInstr
var nilBlk B
vs .defBlk = nilBlk
vs .spilled = false
vs .lastUse = -1
vs .lastUseUpdatedAtBlockID = -1
vs .lca = nilBlk
vs .isPhi = false
vs .phiDefInstList = nil
vs .desiredLoc = desiredLocUnspecified
}
func (s *state [I , B , F ]) getOrAllocateVRegState (v VReg ) *vrState [I , B , F ] {
st := s .vrStates .GetOrAllocate (int (v .ID ()))
if st .v == VRegInvalid {
st .v = v
}
return st
}
func (s *state [I , B , F ]) getVRegState (v VRegID ) *vrState [I , B , F ] {
return s .vrStates .Get (int (v ))
}
func (s *state [I , B , F ]) useRealReg (r RealReg , vr *vrState [I , B , F ]) {
s .regsInUse .add (r , vr )
vr .r = r
s .allocatedRegSet = s .allocatedRegSet .add (r )
}
func (s *state [I , B , F ]) releaseRealReg (r RealReg ) {
current := s .regsInUse .get (r )
if current != nil {
s .regsInUse .remove (r )
current .r = RealRegInvalid
}
}
func (vs *vrState [I , B , F ]) recordReload (f F , blk B ) {
vs .spilled = true
var nilBlk B
if lca := vs .lca ; lca == nilBlk {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\t\tv%d is reloaded in blk%d,\n" , vs .v .ID (), blk .ID ())
}
vs .lca = blk
} else if lca != blk {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\t\tv%d is reloaded in blk%d, lca=%d\n" , vs .v .ID (), blk .ID (), vs .lca .ID ())
}
vs .lca = f .LowestCommonAncestor (lca , blk )
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("updated lca=%d\n" , vs .lca .ID ())
}
}
}
func (a *Allocator [I , B , F ]) findOrSpillAllocatable (s *state [I , B , F ], allocatable []RealReg , forbiddenMask RegSet , preferred RealReg ) (r RealReg ) {
r = RealRegInvalid
if preferred != RealRegInvalid && !forbiddenMask .has (preferred ) && !s .regsInUse .has (preferred ) {
return preferred
}
var lastUseAt programCounter
var spillVReg VReg
for _ , candidateReal := range allocatable {
if forbiddenMask .has (candidateReal ) {
continue
}
using := s .regsInUse .get (candidateReal )
if using == nil {
return candidateReal
}
if using .v .IsRealReg () {
continue
}
isPreferred := candidateReal == preferred
if last := using .lastUse ; r == RealRegInvalid || isPreferred || last == -1 || (lastUseAt != -1 && last > lastUseAt ) {
lastUseAt = last
r = candidateReal
spillVReg = using .v
if isPreferred {
break
}
}
}
if r == RealRegInvalid {
panic ("not found any allocatable register" )
}
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\tspilling v%d when lastUseAt=%d and regsInUse=%s\n" , spillVReg .ID (), lastUseAt , s .regsInUse .format (a .regInfo ))
}
s .releaseRealReg (r )
return r
}
func (s *state [I , B , F ]) findAllocatable (allocatable []RealReg , forbiddenMask RegSet ) RealReg {
for _ , r := range allocatable {
if !s .regsInUse .has (r ) && !forbiddenMask .has (r ) {
return r
}
}
return RealRegInvalid
}
func (s *state [I , B , F ]) resetAt (bs *blockState [I , B , F ]) {
s .regsInUse .range_ (func (_ RealReg , vs *vrState [I , B , F ]) {
vs .r = RealRegInvalid
})
s .regsInUse .reset ()
bs .endRegs .range_ (func (r RealReg , vs *vrState [I , B , F ]) {
if vs .lastUseUpdatedAtBlockID == s .currentBlockID && vs .lastUse == programCounterLiveIn {
s .regsInUse .add (r , vs )
vs .r = r
}
})
}
func resetBlockState[I Instr , B Block [I ], F Function [I , B ]](b *blockState [I , B , F ]) {
b .seen = false
b .visited = false
b .endRegs .reset ()
b .startRegs .reset ()
b .startFromPredIndex = -1
b .liveIns = b .liveIns [:0 ]
}
func (b *blockState [I , B , F ]) dump (a *RegisterInfo ) {
fmt .Println ("\t\tblockState:" )
fmt .Println ("\t\t\tstartRegs:" , b .startRegs .format (a ))
fmt .Println ("\t\t\tendRegs:" , b .endRegs .format (a ))
fmt .Println ("\t\t\tstartFromPredIndex:" , b .startFromPredIndex )
fmt .Println ("\t\t\tvisited:" , b .visited )
}
func (a *Allocator [I , B , F ]) DoAllocation (f F ) {
a .livenessAnalysis (f )
a .alloc (f )
a .determineCalleeSavedRealRegs (f )
}
func (a *Allocator [I , B , F ]) determineCalleeSavedRealRegs (f F ) {
a .allocatedCalleeSavedRegs = a .allocatedCalleeSavedRegs [:0 ]
a .state .allocatedRegSet .Range (func (allocatedRealReg RealReg ) {
if a .regInfo .CalleeSavedRegisters .has (allocatedRealReg ) {
a .allocatedCalleeSavedRegs = append (a .allocatedCalleeSavedRegs , a .regInfo .RealRegToVReg [allocatedRealReg ])
}
})
f .ClobberedRegisters (a .allocatedCalleeSavedRegs )
}
func (a *Allocator [I , B , F ]) getOrAllocateBlockState (blockID int32 ) *blockState [I , B , F ] {
return a .blockStates .GetOrAllocate (int (blockID ))
}
func (vs *vrState [I , B , F ]) phiBlk () B {
if vs .isPhi {
return vs .defBlk
}
var nilBlk B
return nilBlk
}
const (
programCounterLiveIn = math .MinInt32
programCounterLiveOut = math .MaxInt32
)
func (a *Allocator [I , B , F ]) livenessAnalysis (f F ) {
s := &a .state
for i := VRegID (0 ); i < vRegIDReservedForRealNum ; i ++ {
s .getOrAllocateVRegState (VReg (i ).SetRealReg (RealReg (i )))
}
var nilBlk B
var nilInstr I
for blk := f .PostOrderBlockIteratorBegin (); blk != nilBlk ; blk = f .PostOrderBlockIteratorNext () {
for _ , p := range f .BlockParams (blk , &a .vs ) {
vs := s .getOrAllocateVRegState (p )
vs .isPhi = true
vs .defBlk = blk
}
blkID := blk .ID ()
info := a .getOrAllocateBlockState (blkID )
a .ss = a .ss [:0 ]
const (
flagDeleted = false
flagLive = true
)
ns := blk .Succs ()
for i := 0 ; i < ns ; i ++ {
succ := f .Succ (blk , i )
if succ == nilBlk {
continue
}
succID := succ .ID ()
succInfo := a .getOrAllocateBlockState (succID )
if !succInfo .seen {
continue
}
for _ , st := range succInfo .liveIns {
if st .phiBlk () != succ && st .spilled != flagLive {
st .spilled = flagLive
a .ss = append (a .ss , st )
}
}
}
for instr := blk .InstrRevIteratorBegin (); instr != nilInstr ; instr = blk .InstrRevIteratorNext () {
var use , def VReg
var defIsPhi bool
for _, def = range instr .Defs (&a .vs ) {
if !def .IsRealReg () {
st := s .getOrAllocateVRegState (def )
defIsPhi = st .isPhi
st .spilled = flagDeleted
}
}
for _, use = range instr .Uses (&a .vs ) {
if !use .IsRealReg () {
st := s .getOrAllocateVRegState (use )
if st .spilled != flagLive {
st .spilled = flagLive
a .ss = append (a .ss , st )
}
}
}
if defIsPhi {
if use .Valid () && use .IsRealReg () {
a .state .argRealRegs = append (a .state .argRealRegs , use )
}
}
}
for _ , st := range a .ss {
if st .spilled == flagLive {
info .liveIns = append (info .liveIns , st )
st .spilled = false
}
}
info .seen = true
}
nrs := f .LoopNestingForestRoots ()
for i := 0 ; i < nrs ; i ++ {
root := f .LoopNestingForestRoot (i )
a .loopTreeDFS (f , root )
}
}
func (a *Allocator [I , B , F ]) loopTreeDFS (f F , entry B ) {
a .blks = a .blks [:0 ]
a .blks = append (a .blks , entry )
for len (a .blks ) > 0 {
tail := len (a .blks ) - 1
loop := a .blks [tail ]
a .blks = a .blks [:tail ]
a .ss = a .ss [:0 ]
const (
flagDone = false
flagPending = true
)
info := a .getOrAllocateBlockState (loop .ID ())
for _ , st := range info .liveIns {
if st .phiBlk () != loop {
a .ss = append (a .ss , st )
st .spilled = flagPending
}
}
var siblingAddedView []*vrState [I , B , F ]
cn := loop .LoopNestingForestChildren ()
for i := 0 ; i < cn ; i ++ {
child := f .LoopNestingForestChild (loop , i )
childID := child .ID ()
childInfo := a .getOrAllocateBlockState (childID )
if i == 0 {
begin := len (childInfo .liveIns )
for _ , st := range a .ss {
if st .spilled == flagPending {
st .spilled = flagDone
childInfo .liveIns = append (childInfo .liveIns , st )
}
}
siblingAddedView = childInfo .liveIns [begin :]
} else {
childInfo .liveIns = append (childInfo .liveIns , siblingAddedView ...)
}
if child .LoopHeader () {
a .blks = append (a .blks , child )
}
}
if cn == 0 {
for _ , st := range a .ss {
st .spilled = false
}
}
}
}
func (a *Allocator [I , B , F ]) alloc (f F ) {
var nilBlk B
for blk := f .ReversePostOrderBlockIteratorBegin (); blk != nilBlk ; blk = f .ReversePostOrderBlockIteratorNext () {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("========== allocating blk%d ========\n" , blk .ID ())
}
if blk .Entry () {
a .finalizeStartReg (f , blk )
}
a .allocBlock (f , blk )
}
for blk := f .ReversePostOrderBlockIteratorBegin (); blk != nilBlk ; blk = f .ReversePostOrderBlockIteratorNext () {
a .fixMergeState (f , blk )
}
a .scheduleSpills (f )
}
func (a *Allocator [I , B , F ]) updateLiveInVRState (liveness *blockState [I , B , F ]) {
currentBlockID := a .state .currentBlockID
for _ , vs := range liveness .liveIns {
vs .lastUse = programCounterLiveIn
vs .lastUseUpdatedAtBlockID = currentBlockID
}
}
func (a *Allocator [I , B , F ]) finalizeStartReg (f F , blk B ) {
bID := blk .ID ()
s := &a .state
currentBlkState := a .getOrAllocateBlockState (bID )
if currentBlkState .startFromPredIndex > -1 {
return
}
s .currentBlockID = bID
a .updateLiveInVRState (currentBlkState )
preds := blk .Preds ()
var predState *blockState [I , B , F ]
switch preds {
case 0 :
case 1 :
predID := f .Pred (blk , 0 ).ID ()
predState = a .getOrAllocateBlockState (predID )
currentBlkState .startFromPredIndex = 0
default :
for i := 0 ; i < preds ; i ++ {
predID := f .Pred (blk , i ).ID ()
if _predState := a .getOrAllocateBlockState (predID ); _predState .visited {
predState = _predState
currentBlkState .startFromPredIndex = i
break
}
}
}
if predState == nil {
if !blk .Entry () {
panic (fmt .Sprintf ("BUG: at lease one predecessor should be visited for blk%d" , blk .ID ()))
}
for _ , u := range s .argRealRegs {
s .useRealReg (u .RealReg (), s .getVRegState (u .ID ()))
}
currentBlkState .startFromPredIndex = 0
} else {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("allocating blk%d starting from blk%d (on index=%d) \n" ,
bID , f .Pred (blk , currentBlkState .startFromPredIndex ).ID (), currentBlkState .startFromPredIndex )
}
s .resetAt (predState )
}
s .regsInUse .range_ (func (allocated RealReg , v *vrState [I , B , F ]) {
currentBlkState .startRegs .add (allocated , v )
})
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("finalized start reg for blk%d: %s\n" , blk .ID (), currentBlkState .startRegs .format (a .regInfo ))
}
}
func (a *Allocator [I , B , F ]) allocBlock (f F , blk B ) {
bID := blk .ID ()
s := &a .state
currentBlkState := a .getOrAllocateBlockState (bID )
s .currentBlockID = bID
if currentBlkState .startFromPredIndex < 0 {
panic ("BUG: startFromPredIndex should be set in finalizeStartReg prior to allocBlock" )
}
s .regsInUse .range_ (func (allocatedRealReg RealReg , vr *vrState [I , B , F ]) { vr .r = RealRegInvalid })
s .regsInUse .reset ()
currentBlkState .startRegs .range_ (func (allocatedRealReg RealReg , vr *vrState [I , B , F ]) { s .useRealReg (allocatedRealReg , vr ) })
desiredUpdated := a .ss [:0 ]
a .copies = a .copies [:0 ]
var pc programCounter
var nilInstr I
for instr := blk .InstrIteratorBegin (); instr != nilInstr ; instr = blk .InstrIteratorNext () {
var useState *vrState [I , B , F ]
for _ , use := range instr .Uses (&a .vs ) {
useState = s .getVRegState (use .ID ())
if !use .IsRealReg () {
useState .lastUse = pc
}
}
if instr .IsCopy () {
def := instr .Defs (&a .vs )[0 ]
a .copies = append (a .copies , _copy [I , B , F ]{src : useState , dstID : def .ID ()})
r := def .RealReg ()
if r != RealRegInvalid {
if !useState .isPhi {
useState .desiredLoc = newDesiredLocReg (r )
desiredUpdated = append (desiredUpdated , useState )
}
}
}
pc ++
}
var succ B
var nilBlk B
for i , ns := 0 , blk .Succs (); i < ns ; i ++ {
succ = f .Succ (blk , i )
if succ == nilBlk {
continue
}
succID := succ .ID ()
succState := a .getOrAllocateBlockState (succID )
for _ , st := range succState .liveIns {
if st .phiBlk () != succ {
st .lastUse = programCounterLiveOut
}
}
if succState .startFromPredIndex > -1 {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("blk%d -> blk%d: start_regs: %s\n" , bID , succID , succState .startRegs .format (a .regInfo ))
}
succState .startRegs .range_ (func (allocatedRealReg RealReg , vs *vrState [I , B , F ]) {
vs .desiredLoc = newDesiredLocReg (allocatedRealReg )
desiredUpdated = append (desiredUpdated , vs )
})
for _ , p := range f .BlockParams (succ , &a .vs ) {
vs := s .getVRegState (p .ID ())
if vs .desiredLoc .realReg () == RealRegInvalid {
vs .desiredLoc = desiredLocStack
desiredUpdated = append (desiredUpdated , vs )
}
}
}
}
for _ , instr := range a .copies {
defState := s .getVRegState (instr .dstID )
desired := defState .desiredLoc .realReg ()
useState := instr .src
if useState .phiBlk () != succ && useState .desiredLoc == desiredLocUnspecified {
useState .desiredLoc = newDesiredLocReg (desired )
desiredUpdated = append (desiredUpdated , useState )
}
}
pc = 0
for instr := blk .InstrIteratorBegin (); instr != nilInstr ; instr = blk .InstrIteratorNext () {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Println (instr )
}
var currentUsedSet RegSet
killSet := a .reals [:0 ]
uses := instr .Uses (&a .vs )
for _ , use := range uses {
if use .IsRealReg () {
r := use .RealReg ()
currentUsedSet = currentUsedSet .add (r )
if a .allocatableSet .has (r ) {
killSet = append (killSet , r )
}
} else {
vs := s .getVRegState (use .ID ())
if r := vs .r ; r != RealRegInvalid {
currentUsedSet = currentUsedSet .add (r )
}
}
}
for i , use := range uses {
if !use .IsRealReg () {
vs := s .getVRegState (use .ID ())
killed := vs .lastUse == pc
r := vs .r
if r == RealRegInvalid {
r = a .findOrSpillAllocatable (s , a .regInfo .AllocatableRegisters [use .RegType ()], currentUsedSet ,
vs .desiredLoc .realReg ())
vs .recordReload (f , blk )
f .ReloadRegisterBefore (use .SetRealReg (r ), instr )
s .useRealReg (r , vs )
}
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\ttrying to use v%v on %s\n" , use .ID (), a .regInfo .RealRegName (r ))
}
instr .AssignUse (i , use .SetRealReg (r ))
currentUsedSet = currentUsedSet .add (r )
if killed {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\tkill v%d with %s\n" , use .ID (), a .regInfo .RealRegName (r ))
}
killSet = append (killSet , r )
}
}
}
isIndirect := instr .IsIndirectCall ()
if instr .IsCall () || isIndirect {
addr := RealRegInvalid
if isIndirect {
addr = a .vs [0 ].RealReg ()
}
a .releaseCallerSavedRegs (addr )
}
for _ , r := range killSet {
s .releaseRealReg (r )
}
a .reals = killSet
defs := instr .Defs (&a .vs )
switch len (defs ) {
default :
for _ , def := range defs {
if !def .IsRealReg () {
panic ("BUG: multiple defs should be on real registers" )
}
r := def .RealReg ()
if s .regsInUse .has (r ) {
s .releaseRealReg (r )
}
s .useRealReg (r , s .getVRegState (def .ID ()))
}
case 0 :
case 1 :
def := defs [0 ]
vState := s .getVRegState (def .ID ())
if def .IsRealReg () {
r := def .RealReg ()
if a .allocatableSet .has (r ) {
if s .regsInUse .has (r ) {
s .releaseRealReg (r )
}
s .useRealReg (r , vState )
}
} else {
r := vState .r
if desired := vState .desiredLoc .realReg (); desired != RealRegInvalid {
if r != desired {
if (vState .isPhi && vState .defBlk == succ ) ||
(!s .regsInUse .has (desired ) && r == RealRegInvalid ) {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\t\tv%d is phi and desiredReg=%s\n" , def .ID (), a .regInfo .RealRegName (desired ))
}
if r != RealRegInvalid {
s .releaseRealReg (r )
}
r = desired
s .releaseRealReg (r )
s .useRealReg (r , vState )
}
}
}
if r == RealRegInvalid {
if instr .IsCopy () {
copySrc := instr .Uses (&a .vs )[0 ].RealReg ()
if a .allocatableSet .has (copySrc ) && !s .regsInUse .has (copySrc ) {
r = copySrc
}
}
if r == RealRegInvalid {
typ := def .RegType ()
r = a .findOrSpillAllocatable (s , a .regInfo .AllocatableRegisters [typ ], RegSet (0 ), RealRegInvalid )
}
s .useRealReg (r , vState )
}
dr := def .SetRealReg (r )
instr .AssignDef (dr )
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\tdefining v%d with %s\n" , def .ID (), a .regInfo .RealRegName (r ))
}
if vState .isPhi {
if vState .desiredLoc .stack () {
f .StoreRegisterAfter (dr , instr )
s .releaseRealReg (r )
} else {
n := a .phiDefInstListPool .Allocate ()
n .instr = instr
n .next = vState .phiDefInstList
n .v = dr
vState .phiDefInstList = n
}
} else {
vState .defInstr = instr
vState .defBlk = blk
}
}
}
if wazevoapi .RegAllocLoggingEnabled {
fmt .Println (instr )
}
pc ++
}
s .regsInUse .range_ (func (allocated RealReg , v *vrState [I , B , F ]) { currentBlkState .endRegs .add (allocated , v ) })
currentBlkState .visited = true
if wazevoapi .RegAllocLoggingEnabled {
currentBlkState .dump (a .regInfo )
}
for _ , vs := range desiredUpdated {
vs .desiredLoc = desiredLocUnspecified
}
a .ss = desiredUpdated [:0 ]
for i := 0 ; i < blk .Succs (); i ++ {
succ := f .Succ (blk , i )
if succ == nilBlk {
continue
}
a .finalizeStartReg (f , succ )
}
}
func (a *Allocator [I , B , F ]) releaseCallerSavedRegs (addrReg RealReg ) {
s := &a .state
for allocated := RealReg (0 ); allocated < 64 ; allocated ++ {
if allocated == addrReg {
continue
}
if vs := s .regsInUse .get (allocated ); vs != nil {
if vs .v .IsRealReg () {
continue
}
if !a .regInfo .CallerSavedRegisters .has (allocated ) {
continue
}
s .releaseRealReg (allocated )
}
}
}
func (a *Allocator [I , B , F ]) fixMergeState (f F , blk B ) {
preds := blk .Preds ()
if preds <= 1 {
return
}
s := &a .state
bID := blk .ID ()
blkSt := a .getOrAllocateBlockState (bID )
desiredOccupants := &blkSt .startRegs
var desiredOccupantsSet RegSet
for i , v := range desiredOccupants {
if v != nil {
desiredOccupantsSet = desiredOccupantsSet .add (RealReg (i ))
}
}
if wazevoapi .RegAllocLoggingEnabled {
fmt .Println ("fixMergeState" , blk .ID (), ":" , desiredOccupants .format (a .regInfo ))
}
s .currentBlockID = bID
a .updateLiveInVRState (blkSt )
for i := 0 ; i < preds ; i ++ {
if i == blkSt .startFromPredIndex {
continue
}
pred := f .Pred (blk , i )
predSt := a .getOrAllocateBlockState (pred .ID ())
s .resetAt (predSt )
intTmp , floatTmp := VRegInvalid , VRegInvalid
if intFree := s .findAllocatable (
a .regInfo .AllocatableRegisters [RegTypeInt ], desiredOccupantsSet ,
); intFree != RealRegInvalid {
intTmp = FromRealReg (intFree , RegTypeInt )
}
if floatFree := s .findAllocatable (
a .regInfo .AllocatableRegisters [RegTypeFloat ], desiredOccupantsSet ,
); floatFree != RealRegInvalid {
floatTmp = FromRealReg (floatFree , RegTypeFloat )
}
for r := RealReg (0 ); r < 64 ; r ++ {
desiredVReg := desiredOccupants .get (r )
if desiredVReg == nil {
continue
}
currentVReg := s .regsInUse .get (r )
if currentVReg != nil && desiredVReg .v .ID () == currentVReg .v .ID () {
continue
}
typ := desiredVReg .v .RegType ()
var tmpRealReg VReg
if typ == RegTypeInt {
tmpRealReg = intTmp
} else {
tmpRealReg = floatTmp
}
a .reconcileEdge (f , r , pred , currentVReg , desiredVReg , tmpRealReg , typ )
}
}
}
func (a *Allocator [I , B , F ]) reconcileEdge (f F ,
r RealReg ,
pred B ,
currentState , desiredState *vrState [I , B , F ],
freeReg VReg ,
typ RegType ,
) {
desiredVReg := desiredState .v
currentVReg := VRegInvalid
if currentState != nil {
currentVReg = currentState .v
}
s := &a .state
if currentVReg .Valid () {
er := desiredState .r
if er == RealRegInvalid {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\t\tv%d is desired to be on %s, but currently on the stack\n" ,
desiredVReg .ID (), a .regInfo .RealRegName (r ),
)
}
f .StoreRegisterBefore (currentVReg .SetRealReg (r ), pred .LastInstrForInsertion ())
s .releaseRealReg (r )
desiredState .recordReload (f , pred )
f .ReloadRegisterBefore (desiredVReg .SetRealReg (r ), pred .LastInstrForInsertion ())
s .useRealReg (r , desiredState )
return
} else {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\t\tv%d is desired to be on %s, but currently on %s\n" ,
desiredVReg .ID (), a .regInfo .RealRegName (r ), a .regInfo .RealRegName (er ),
)
}
f .SwapBefore (
currentVReg .SetRealReg (r ),
desiredVReg .SetRealReg (er ),
freeReg ,
pred .LastInstrForInsertion (),
)
s .allocatedRegSet = s .allocatedRegSet .add (freeReg .RealReg ())
s .releaseRealReg (r )
s .releaseRealReg (er )
s .useRealReg (r , desiredState )
s .useRealReg (er , currentState )
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\t\tv%d previously on %s moved to %s\n" , currentVReg .ID (), a .regInfo .RealRegName (r ), a .regInfo .RealRegName (er ))
}
}
} else {
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("\t\tv%d is desired to be on %s, current not used\n" ,
desiredVReg .ID (), a .regInfo .RealRegName (r ),
)
}
if currentReg := desiredState .r ; currentReg != RealRegInvalid {
f .InsertMoveBefore (
FromRealReg (r , typ ),
desiredVReg .SetRealReg (currentReg ),
pred .LastInstrForInsertion (),
)
s .releaseRealReg (currentReg )
} else {
desiredState .recordReload (f , pred )
f .ReloadRegisterBefore (desiredVReg .SetRealReg (r ), pred .LastInstrForInsertion ())
}
s .useRealReg (r , desiredState )
}
}
func (a *Allocator [I , B , F ]) scheduleSpills (f F ) {
states := a .state .vrStates
for i := 0 ; i <= states .MaxIDEncountered (); i ++ {
vs := states .Get (i )
if vs == nil {
continue
}
if vs .spilled {
a .scheduleSpill (f , vs )
}
}
}
func (a *Allocator [I , B , F ]) scheduleSpill (f F , vs *vrState [I , B , F ]) {
v := vs .v
if vs .isPhi {
for defInstr := vs .phiDefInstList ; defInstr != nil ; defInstr = defInstr .next {
f .StoreRegisterAfter (defInstr .v , defInstr .instr )
}
return
}
pos := vs .lca
definingBlk := vs .defBlk
r := RealRegInvalid
var nilBlk B
if definingBlk == nilBlk {
panic (fmt .Sprintf ("BUG: definingBlk should not be nil for %s. This is likley a bug in backend lowering logic" , vs .v .String ()))
}
if pos == nilBlk {
panic (fmt .Sprintf ("BUG: pos should not be nil for %s. This is likley a bug in backend lowering logic" , vs .v .String ()))
}
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("v%d is spilled in blk%d, lca=blk%d\n" , v .ID (), definingBlk .ID (), pos .ID ())
}
for pos != definingBlk {
st := a .getOrAllocateBlockState (pos .ID ())
for rr := RealReg (0 ); rr < 64 ; rr ++ {
if vs := st .startRegs .get (rr ); vs != nil && vs .v == v {
r = rr
break
}
}
if r != RealRegInvalid {
break
}
pos = f .Idom (pos )
}
if pos == definingBlk {
defInstr := vs .defInstr
defInstr .Defs (&a .vs )
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("schedule spill v%d after %v\n" , v .ID (), defInstr )
}
f .StoreRegisterAfter (a .vs [0 ], defInstr )
} else {
first := pos .FirstInstr ()
if wazevoapi .RegAllocLoggingEnabled {
fmt .Printf ("schedule spill v%d before %v\n" , v .ID (), first )
}
f .StoreRegisterAfter (v .SetRealReg (r ), first )
}
}
func (a *Allocator [I , B , F ]) Reset () {
a .state .reset ()
a .blockStates .Reset ()
a .phiDefInstListPool .Reset ()
a .vs = a .vs [:0 ]
}
The pages are generated with Golds v0.8.2 . (GOOS=linux GOARCH=amd64)
Golds is a Go 101 project developed by Tapir Liu .
PR and bug reports are welcome and can be submitted to the issue list .
Please follow @zigo_101 (reachable from the left QR code) to get the latest news of Golds .