package runtime
import (
"internal/goarch"
"internal/goexperiment"
"internal/runtime/sys"
"unsafe"
)
func mallocgcSmallScanNoHeaderSC1(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (1 <<1 ) | spanClass (0 )
const elemsize = uintptr (8 )
return mallocgcSmallScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 1
const elemsize = 8
if doubleCheckMalloc {
doubleCheckSmallScanNoHeader (size , typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (0 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
if goarch .PtrSize == 8 && elemsize == 8 {
c .scanAlloc += 8
} else {
dataSize := size
x := uintptr (x )
if doubleCheckHeapSetType && (!heapBitsInSpan (dataSize ) || !heapBitsInSpan (8 )) {
throw ("tried to write heap bits, but no heap bits in span" )
}
src0 := readUintptr (getGCMask (typ ))
const elemsize = 8
var scanSize uintptr
src := src0
if typ .Size_ == goarch .PtrSize {
src = (1 << (dataSize / goarch .PtrSize )) - 1
scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize %typ .Size_ != 0 {
throw ("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_" )
}
scanSize = typ .PtrBytes
for i := typ .Size_ ; i < dataSize ; i += typ .Size_ {
src |= src0 << (i / goarch .PtrSize )
scanSize += typ .Size_
}
}
dstBase , _ := spanHeapBitsRange (span .base (), pageSize , elemsize )
dst := unsafe .Pointer (dstBase )
o := (x - span .base ()) / goarch .PtrSize
i := o / ptrBits
j := o % ptrBits
var bits uintptr = elemsize / goarch .PtrSize
var bitsIsPowerOfTwo = bits &(bits -1 ) == 0
if bits > ptrBits || (!bitsIsPowerOfTwo && j +bits > ptrBits ) {
bits0 := ptrBits - j
bits1 := bits - bits0
dst0 := (*uintptr )(add (dst , (i +0 )*goarch .PtrSize ))
dst1 := (*uintptr )(add (dst , (i +1 )*goarch .PtrSize ))
*dst0 = (*dst0 )&(^uintptr (0 )>>bits0 ) | (src << j )
*dst1 = (*dst1 )&^((1 <<bits1 )-1 ) | (src >> bits0 )
} else {
dst := (*uintptr )(add (dst , i *goarch .PtrSize ))
*dst = (*dst )&^(((1 <<(min (bits , ptrBits )))-1 )<<j ) | (src << j )
}
const doubleCheck = false
if doubleCheck {
writeHeapBitsDoubleCheck (span , x , dataSize , src , src0 , i , j , bits , typ )
}
if doubleCheckHeapSetType {
doubleCheckHeapType (x , dataSize , typ , nil , span )
}
c .scanAlloc += scanSize
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallScanNoHeaderSC2(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (2 <<1 ) | spanClass (0 )
const elemsize = uintptr (16 )
return mallocgcSmallScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 2
const elemsize = 16
if doubleCheckMalloc {
doubleCheckSmallScanNoHeader (size , typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (0 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
if goarch .PtrSize == 8 && elemsize == 8 {
c .scanAlloc += 8
} else {
dataSize := size
x := uintptr (x )
if doubleCheckHeapSetType && (!heapBitsInSpan (dataSize ) || !heapBitsInSpan (16 )) {
throw ("tried to write heap bits, but no heap bits in span" )
}
src0 := readUintptr (getGCMask (typ ))
const elemsize = 16
var scanSize uintptr
src := src0
if typ .Size_ == goarch .PtrSize {
src = (1 << (dataSize / goarch .PtrSize )) - 1
scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize %typ .Size_ != 0 {
throw ("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_" )
}
scanSize = typ .PtrBytes
for i := typ .Size_ ; i < dataSize ; i += typ .Size_ {
src |= src0 << (i / goarch .PtrSize )
scanSize += typ .Size_
}
}
dstBase , _ := spanHeapBitsRange (span .base (), pageSize , elemsize )
dst := unsafe .Pointer (dstBase )
o := (x - span .base ()) / goarch .PtrSize
i := o / ptrBits
j := o % ptrBits
var bits uintptr = elemsize / goarch .PtrSize
var bitsIsPowerOfTwo = bits &(bits -1 ) == 0
if bits > ptrBits || (!bitsIsPowerOfTwo && j +bits > ptrBits ) {
bits0 := ptrBits - j
bits1 := bits - bits0
dst0 := (*uintptr )(add (dst , (i +0 )*goarch .PtrSize ))
dst1 := (*uintptr )(add (dst , (i +1 )*goarch .PtrSize ))
*dst0 = (*dst0 )&(^uintptr (0 )>>bits0 ) | (src << j )
*dst1 = (*dst1 )&^((1 <<bits1 )-1 ) | (src >> bits0 )
} else {
dst := (*uintptr )(add (dst , i *goarch .PtrSize ))
*dst = (*dst )&^(((1 <<(min (bits , ptrBits )))-1 )<<j ) | (src << j )
}
const doubleCheck = false
if doubleCheck {
writeHeapBitsDoubleCheck (span , x , dataSize , src , src0 , i , j , bits , typ )
}
if doubleCheckHeapSetType {
doubleCheckHeapType (x , dataSize , typ , nil , span )
}
c .scanAlloc += scanSize
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallScanNoHeaderSC3(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (3 <<1 ) | spanClass (0 )
const elemsize = uintptr (24 )
return mallocgcSmallScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 3
const elemsize = 24
if doubleCheckMalloc {
doubleCheckSmallScanNoHeader (size , typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (0 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
if goarch .PtrSize == 8 && elemsize == 8 {
c .scanAlloc += 8
} else {
dataSize := size
x := uintptr (x )
if doubleCheckHeapSetType && (!heapBitsInSpan (dataSize ) || !heapBitsInSpan (24 )) {
throw ("tried to write heap bits, but no heap bits in span" )
}
src0 := readUintptr (getGCMask (typ ))
const elemsize = 24
var scanSize uintptr
src := src0
if typ .Size_ == goarch .PtrSize {
src = (1 << (dataSize / goarch .PtrSize )) - 1
scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize %typ .Size_ != 0 {
throw ("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_" )
}
scanSize = typ .PtrBytes
for i := typ .Size_ ; i < dataSize ; i += typ .Size_ {
src |= src0 << (i / goarch .PtrSize )
scanSize += typ .Size_
}
}
dstBase , _ := spanHeapBitsRange (span .base (), pageSize , elemsize )
dst := unsafe .Pointer (dstBase )
o := (x - span .base ()) / goarch .PtrSize
i := o / ptrBits
j := o % ptrBits
var bits uintptr = elemsize / goarch .PtrSize
var bitsIsPowerOfTwo = bits &(bits -1 ) == 0
if bits > ptrBits || (!bitsIsPowerOfTwo && j +bits > ptrBits ) {
bits0 := ptrBits - j
bits1 := bits - bits0
dst0 := (*uintptr )(add (dst , (i +0 )*goarch .PtrSize ))
dst1 := (*uintptr )(add (dst , (i +1 )*goarch .PtrSize ))
*dst0 = (*dst0 )&(^uintptr (0 )>>bits0 ) | (src << j )
*dst1 = (*dst1 )&^((1 <<bits1 )-1 ) | (src >> bits0 )
} else {
dst := (*uintptr )(add (dst , i *goarch .PtrSize ))
*dst = (*dst )&^(((1 <<(min (bits , ptrBits )))-1 )<<j ) | (src << j )
}
const doubleCheck = false
if doubleCheck {
writeHeapBitsDoubleCheck (span , x , dataSize , src , src0 , i , j , bits , typ )
}
if doubleCheckHeapSetType {
doubleCheckHeapType (x , dataSize , typ , nil , span )
}
c .scanAlloc += scanSize
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallScanNoHeaderSC4(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (4 <<1 ) | spanClass (0 )
const elemsize = uintptr (32 )
return mallocgcSmallScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 4
const elemsize = 32
if doubleCheckMalloc {
doubleCheckSmallScanNoHeader (size , typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (0 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
if goarch .PtrSize == 8 && elemsize == 8 {
c .scanAlloc += 8
} else {
dataSize := size
x := uintptr (x )
if doubleCheckHeapSetType && (!heapBitsInSpan (dataSize ) || !heapBitsInSpan (32 )) {
throw ("tried to write heap bits, but no heap bits in span" )
}
src0 := readUintptr (getGCMask (typ ))
const elemsize = 32
var scanSize uintptr
src := src0
if typ .Size_ == goarch .PtrSize {
src = (1 << (dataSize / goarch .PtrSize )) - 1
scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize %typ .Size_ != 0 {
throw ("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_" )
}
scanSize = typ .PtrBytes
for i := typ .Size_ ; i < dataSize ; i += typ .Size_ {
src |= src0 << (i / goarch .PtrSize )
scanSize += typ .Size_
}
}
dstBase , _ := spanHeapBitsRange (span .base (), pageSize , elemsize )
dst := unsafe .Pointer (dstBase )
o := (x - span .base ()) / goarch .PtrSize
i := o / ptrBits
j := o % ptrBits
var bits uintptr = elemsize / goarch .PtrSize
var bitsIsPowerOfTwo = bits &(bits -1 ) == 0
if bits > ptrBits || (!bitsIsPowerOfTwo && j +bits > ptrBits ) {
bits0 := ptrBits - j
bits1 := bits - bits0
dst0 := (*uintptr )(add (dst , (i +0 )*goarch .PtrSize ))
dst1 := (*uintptr )(add (dst , (i +1 )*goarch .PtrSize ))
*dst0 = (*dst0 )&(^uintptr (0 )>>bits0 ) | (src << j )
*dst1 = (*dst1 )&^((1 <<bits1 )-1 ) | (src >> bits0 )
} else {
dst := (*uintptr )(add (dst , i *goarch .PtrSize ))
*dst = (*dst )&^(((1 <<(min (bits , ptrBits )))-1 )<<j ) | (src << j )
}
const doubleCheck = false
if doubleCheck {
writeHeapBitsDoubleCheck (span , x , dataSize , src , src0 , i , j , bits , typ )
}
if doubleCheckHeapSetType {
doubleCheckHeapType (x , dataSize , typ , nil , span )
}
c .scanAlloc += scanSize
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallScanNoHeaderSC5(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (5 <<1 ) | spanClass (0 )
const elemsize = uintptr (48 )
return mallocgcSmallScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 5
const elemsize = 48
if doubleCheckMalloc {
doubleCheckSmallScanNoHeader (size , typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (0 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
if goarch .PtrSize == 8 && elemsize == 8 {
c .scanAlloc += 8
} else {
dataSize := size
x := uintptr (x )
if doubleCheckHeapSetType && (!heapBitsInSpan (dataSize ) || !heapBitsInSpan (48 )) {
throw ("tried to write heap bits, but no heap bits in span" )
}
src0 := readUintptr (getGCMask (typ ))
const elemsize = 48
var scanSize uintptr
src := src0
if typ .Size_ == goarch .PtrSize {
src = (1 << (dataSize / goarch .PtrSize )) - 1
scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize %typ .Size_ != 0 {
throw ("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_" )
}
scanSize = typ .PtrBytes
for i := typ .Size_ ; i < dataSize ; i += typ .Size_ {
src |= src0 << (i / goarch .PtrSize )
scanSize += typ .Size_
}
}
dstBase , _ := spanHeapBitsRange (span .base (), pageSize , elemsize )
dst := unsafe .Pointer (dstBase )
o := (x - span .base ()) / goarch .PtrSize
i := o / ptrBits
j := o % ptrBits
var bits uintptr = elemsize / goarch .PtrSize
var bitsIsPowerOfTwo = bits &(bits -1 ) == 0
if bits > ptrBits || (!bitsIsPowerOfTwo && j +bits > ptrBits ) {
bits0 := ptrBits - j
bits1 := bits - bits0
dst0 := (*uintptr )(add (dst , (i +0 )*goarch .PtrSize ))
dst1 := (*uintptr )(add (dst , (i +1 )*goarch .PtrSize ))
*dst0 = (*dst0 )&(^uintptr (0 )>>bits0 ) | (src << j )
*dst1 = (*dst1 )&^((1 <<bits1 )-1 ) | (src >> bits0 )
} else {
dst := (*uintptr )(add (dst , i *goarch .PtrSize ))
*dst = (*dst )&^(((1 <<(min (bits , ptrBits )))-1 )<<j ) | (src << j )
}
const doubleCheck = false
if doubleCheck {
writeHeapBitsDoubleCheck (span , x , dataSize , src , src0 , i , j , bits , typ )
}
if doubleCheckHeapSetType {
doubleCheckHeapType (x , dataSize , typ , nil , span )
}
c .scanAlloc += scanSize
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallScanNoHeaderSC6(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (6 <<1 ) | spanClass (0 )
const elemsize = uintptr (64 )
return mallocgcSmallScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 6
const elemsize = 64
if doubleCheckMalloc {
doubleCheckSmallScanNoHeader (size , typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (0 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
if goarch .PtrSize == 8 && elemsize == 8 {
c .scanAlloc += 8
} else {
dataSize := size
x := uintptr (x )
if doubleCheckHeapSetType && (!heapBitsInSpan (dataSize ) || !heapBitsInSpan (64 )) {
throw ("tried to write heap bits, but no heap bits in span" )
}
src0 := readUintptr (getGCMask (typ ))
const elemsize = 64
var scanSize uintptr
src := src0
if typ .Size_ == goarch .PtrSize {
src = (1 << (dataSize / goarch .PtrSize )) - 1
scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize %typ .Size_ != 0 {
throw ("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_" )
}
scanSize = typ .PtrBytes
for i := typ .Size_ ; i < dataSize ; i += typ .Size_ {
src |= src0 << (i / goarch .PtrSize )
scanSize += typ .Size_
}
}
dstBase , _ := spanHeapBitsRange (span .base (), pageSize , elemsize )
dst := unsafe .Pointer (dstBase )
o := (x - span .base ()) / goarch .PtrSize
i := o / ptrBits
j := o % ptrBits
var bits uintptr = elemsize / goarch .PtrSize
var bitsIsPowerOfTwo = bits &(bits -1 ) == 0
if bits > ptrBits || (!bitsIsPowerOfTwo && j +bits > ptrBits ) {
bits0 := ptrBits - j
bits1 := bits - bits0
dst0 := (*uintptr )(add (dst , (i +0 )*goarch .PtrSize ))
dst1 := (*uintptr )(add (dst , (i +1 )*goarch .PtrSize ))
*dst0 = (*dst0 )&(^uintptr (0 )>>bits0 ) | (src << j )
*dst1 = (*dst1 )&^((1 <<bits1 )-1 ) | (src >> bits0 )
} else {
dst := (*uintptr )(add (dst , i *goarch .PtrSize ))
*dst = (*dst )&^(((1 <<(min (bits , ptrBits )))-1 )<<j ) | (src << j )
}
const doubleCheck = false
if doubleCheck {
writeHeapBitsDoubleCheck (span , x , dataSize , src , src0 , i , j , bits , typ )
}
if doubleCheckHeapSetType {
doubleCheckHeapType (x , dataSize , typ , nil , span )
}
c .scanAlloc += scanSize
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallScanNoHeaderSC7(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (7 <<1 ) | spanClass (0 )
const elemsize = uintptr (80 )
return mallocgcSmallScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 7
const elemsize = 80
if doubleCheckMalloc {
doubleCheckSmallScanNoHeader (size , typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (0 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
if goarch .PtrSize == 8 && elemsize == 8 {
c .scanAlloc += 8
} else {
dataSize := size
x := uintptr (x )
if doubleCheckHeapSetType && (!heapBitsInSpan (dataSize ) || !heapBitsInSpan (80 )) {
throw ("tried to write heap bits, but no heap bits in span" )
}
src0 := readUintptr (getGCMask (typ ))
const elemsize = 80
var scanSize uintptr
src := src0
if typ .Size_ == goarch .PtrSize {
src = (1 << (dataSize / goarch .PtrSize )) - 1
scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize %typ .Size_ != 0 {
throw ("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_" )
}
scanSize = typ .PtrBytes
for i := typ .Size_ ; i < dataSize ; i += typ .Size_ {
src |= src0 << (i / goarch .PtrSize )
scanSize += typ .Size_
}
}
dstBase , _ := spanHeapBitsRange (span .base (), pageSize , elemsize )
dst := unsafe .Pointer (dstBase )
o := (x - span .base ()) / goarch .PtrSize
i := o / ptrBits
j := o % ptrBits
var bits uintptr = elemsize / goarch .PtrSize
var bitsIsPowerOfTwo = bits &(bits -1 ) == 0
if bits > ptrBits || (!bitsIsPowerOfTwo && j +bits > ptrBits ) {
bits0 := ptrBits - j
bits1 := bits - bits0
dst0 := (*uintptr )(add (dst , (i +0 )*goarch .PtrSize ))
dst1 := (*uintptr )(add (dst , (i +1 )*goarch .PtrSize ))
*dst0 = (*dst0 )&(^uintptr (0 )>>bits0 ) | (src << j )
*dst1 = (*dst1 )&^((1 <<bits1 )-1 ) | (src >> bits0 )
} else {
dst := (*uintptr )(add (dst , i *goarch .PtrSize ))
*dst = (*dst )&^(((1 <<(min (bits , ptrBits )))-1 )<<j ) | (src << j )
}
const doubleCheck = false
if doubleCheck {
writeHeapBitsDoubleCheck (span , x , dataSize , src , src0 , i , j , bits , typ )
}
if doubleCheckHeapSetType {
doubleCheckHeapType (x , dataSize , typ , nil , span )
}
c .scanAlloc += scanSize
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcTinySC2(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
return mallocgcTinySlowPath (size , typ , needzero )
}
lockRankMayQueueFinalizer ()
const elemsize = 16
if doubleCheckMalloc {
doubleCheckTiny (size , typ , mp )
}
mp .mallocing = 1
c := getMCache (mp )
off := c .tinyoffset
if size &7 == 0 {
off = alignUp (off , 8 )
} else if goarch .PtrSize == 4 && size == 12 {
off = alignUp (off , 8 )
} else if size &3 == 0 {
off = alignUp (off , 4 )
} else if size &1 == 0 {
off = alignUp (off , 2 )
}
if off +size <= maxTinySize && c .tiny != 0 {
x := unsafe .Pointer (c .tiny + off )
c .tinyoffset = off + size
c .tinyAllocs ++
mp .mallocing = 0
releasem (mp )
const elemsize = 0
return x
}
checkGCTrigger := false
span := c .alloc [tinySpanClass ]
const nbytes = 8192
const nelems = uint16 ((nbytes - unsafe .Sizeof (spanInlineMarkBits {})) / 16 )
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*16 + span .base ())
}
}
}
v := nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (tinySpanClass )
}
x := unsafe .Pointer (v )
(*[2 ]uint64 )(x )[0 ] = 0
(*[2 ]uint64 )(x )[1 ] = 0
if !raceenabled && (size < c .tinyoffset || c .tiny == 0 ) {
c .tiny = uintptr (x )
c .tinyoffset = size
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallNoScanSC2(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (2 <<1 ) | spanClass (1 )
const elemsize = uintptr (16 )
return mallocgcSmallNoScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 2
const elemsize = 16
if doubleCheckMalloc {
doubleCheckSmallNoScan (typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (1 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
if runtimeFreegcEnabled && c .hasReusableNoscan (spc ) {
x = mallocgcSmallNoscanReuse (c , span , spc , elemsize , needzero )
mp .mallocing = 0
releasem (mp )
return x
}
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if needzero && span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallNoScanSC3(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (3 <<1 ) | spanClass (1 )
const elemsize = uintptr (24 )
return mallocgcSmallNoScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 3
const elemsize = 24
if doubleCheckMalloc {
doubleCheckSmallNoScan (typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (1 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
if runtimeFreegcEnabled && c .hasReusableNoscan (spc ) {
x = mallocgcSmallNoscanReuse (c , span , spc , elemsize , needzero )
mp .mallocing = 0
releasem (mp )
return x
}
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if needzero && span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallNoScanSC4(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (4 <<1 ) | spanClass (1 )
const elemsize = uintptr (32 )
return mallocgcSmallNoScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 4
const elemsize = 32
if doubleCheckMalloc {
doubleCheckSmallNoScan (typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (1 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
if runtimeFreegcEnabled && c .hasReusableNoscan (spc ) {
x = mallocgcSmallNoscanReuse (c , span , spc , elemsize , needzero )
mp .mallocing = 0
releasem (mp )
return x
}
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if needzero && span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallNoScanSC5(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (5 <<1 ) | spanClass (1 )
const elemsize = uintptr (48 )
return mallocgcSmallNoScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 5
const elemsize = 48
if doubleCheckMalloc {
doubleCheckSmallNoScan (typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (1 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
if runtimeFreegcEnabled && c .hasReusableNoscan (spc ) {
x = mallocgcSmallNoscanReuse (c , span , spc , elemsize , needzero )
mp .mallocing = 0
releasem (mp )
return x
}
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if needzero && span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallNoScanSC6(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (6 <<1 ) | spanClass (1 )
const elemsize = uintptr (64 )
return mallocgcSmallNoScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 6
const elemsize = 64
if doubleCheckMalloc {
doubleCheckSmallNoScan (typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (1 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
if runtimeFreegcEnabled && c .hasReusableNoscan (spc ) {
x = mallocgcSmallNoscanReuse (c , span , spc , elemsize , needzero )
mp .mallocing = 0
releasem (mp )
return x
}
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if needzero && span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcSmallNoScanSC7(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
mp = acquirem ()
forceSlowPath := debug .malloc || gcBlackenEnabled != 0 || (goexperiment .RuntimeSecret && getg ().secret > 0 )
if forceSlowPath {
releasem (mp )
const spc = spanClass (7 <<1 ) | spanClass (1 )
const elemsize = uintptr (80 )
return mallocgcSmallNoScanSlowPath (size , typ , needzero , spc , elemsize )
}
lockRankMayQueueFinalizer ()
const sizeclass = 7
const elemsize = 80
if doubleCheckMalloc {
doubleCheckSmallNoScan (typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
const spc = spanClass (sizeclass <<1 ) | spanClass (1 )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
if runtimeFreegcEnabled && c .hasReusableNoscan (spc ) {
x = mallocgcSmallNoscanReuse (c , span , spc , elemsize , needzero )
mp .mallocing = 0
releasem (mp )
return x
}
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if needzero && span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
publicationBarrier ()
span .freeIndexForScan = span .freeindex
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
return x
}
func mallocgcTinySlowPath(size uintptr , typ *_type , needzero bool ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
gp := getg ()
if goexperiment .RuntimeSecret && gp .secret > 0 {
return mallocgcSmallNoScanSC2 (size , typ , needzero )
}
lockRankMayQueueFinalizer ()
if debug .malloc {
if x := preMallocgcDebug (size , typ ); x != nil {
return x
}
}
if gcBlackenEnabled != 0 {
assistG := getg ()
if assistG .m .curg != nil {
assistG = assistG .m .curg
}
assistG .gcAssistBytes -= int64 (size )
if assistG .gcAssistBytes < 0 {
gcAssistAlloc (assistG )
}
}
mp = acquirem ()
const elemsize = 16
if doubleCheckMalloc {
doubleCheckTiny (size , typ , mp )
}
mp .mallocing = 1
c := getMCache (mp )
off := c .tinyoffset
if size &7 == 0 {
off = alignUp (off , 8 )
} else if goarch .PtrSize == 4 && size == 12 {
off = alignUp (off , 8 )
} else if size &3 == 0 {
off = alignUp (off , 4 )
} else if size &1 == 0 {
off = alignUp (off , 2 )
}
if off +size <= maxTinySize && c .tiny != 0 {
x := unsafe .Pointer (c .tiny + off )
c .tinyoffset = off + size
c .tinyAllocs ++
mp .mallocing = 0
releasem (mp )
const elemsize = 0
if gcBlackenEnabled != 0 && elemsize != 0 {
if assistG := getg ().m .curg ; assistG != nil {
assistG .gcAssistBytes -= int64 (elemsize - size )
}
}
if debug .malloc {
postMallocgcDebug (x , elemsize , typ )
}
return x
}
checkGCTrigger := false
span := c .alloc [tinySpanClass ]
const nbytes = 8192
const nelems = uint16 ((nbytes - unsafe .Sizeof (spanInlineMarkBits {})) / 16 )
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*16 + span .base ())
}
}
}
v := nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (tinySpanClass )
}
x := unsafe .Pointer (v )
(*[2 ]uint64 )(x )[0 ] = 0
(*[2 ]uint64 )(x )[1 ] = 0
if !raceenabled && (size < c .tinyoffset || c .tiny == 0 ) {
c .tiny = uintptr (x )
c .tinyoffset = size
}
publicationBarrier ()
if writeBarrier .enabled {
gcmarknewobject (span , uintptr (x ))
} else {
span .freeIndexForScan = span .freeindex
}
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
if gcBlackenEnabled != 0 && elemsize != 0 {
if assistG := getg ().m .curg ; assistG != nil {
assistG .gcAssistBytes -= int64 (elemsize - size )
}
}
if debug .malloc {
postMallocgcDebug (x , elemsize , typ )
}
return x
}
func mallocgcSmallScanSlowPath(size uintptr , typ *_type , needzero bool , spc spanClass , elemsize uintptr ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
lockRankMayQueueFinalizer ()
if debug .malloc {
if x := preMallocgcDebug (size , typ ); x != nil {
return x
}
}
if gcBlackenEnabled != 0 {
assistG := getg ()
if assistG .m .curg != nil {
assistG = assistG .m .curg
}
assistG .gcAssistBytes -= int64 (size )
if assistG .gcAssistBytes < 0 {
gcAssistAlloc (assistG )
}
}
mp = acquirem ()
if doubleCheckMalloc {
doubleCheckSmallScanNoHeader (size , typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
if goarch .PtrSize == 8 && elemsize == 8 {
c .scanAlloc += 8
} else {
dataSize := size
x := uintptr (x )
if doubleCheckHeapSetType && (!heapBitsInSpan (dataSize ) || !heapBitsInSpan (elemsize_ )) {
throw ("tried to write heap bits, but no heap bits in span" )
}
src0 := readUintptr (getGCMask (typ ))
var scanSize uintptr
src := src0
if typ .Size_ == goarch .PtrSize {
src = (1 << (dataSize / goarch .PtrSize )) - 1
scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize %typ .Size_ != 0 {
throw ("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_" )
}
scanSize = typ .PtrBytes
for i := typ .Size_ ; i < dataSize ; i += typ .Size_ {
src |= src0 << (i / goarch .PtrSize )
scanSize += typ .Size_
}
}
dstBase , _ := spanHeapBitsRange (span .base (), pageSize , elemsize )
dst := unsafe .Pointer (dstBase )
o := (x - span .base ()) / goarch .PtrSize
i := o / ptrBits
j := o % ptrBits
var bits uintptr = elemsize / goarch .PtrSize
var bitsIsPowerOfTwo = bits &(bits -1 ) == 0
if bits > ptrBits || (!bitsIsPowerOfTwo && j +bits > ptrBits ) {
bits0 := ptrBits - j
bits1 := bits - bits0
dst0 := (*uintptr )(add (dst , (i +0 )*goarch .PtrSize ))
dst1 := (*uintptr )(add (dst , (i +1 )*goarch .PtrSize ))
*dst0 = (*dst0 )&(^uintptr (0 )>>bits0 ) | (src << j )
*dst1 = (*dst1 )&^((1 <<bits1 )-1 ) | (src >> bits0 )
} else {
dst := (*uintptr )(add (dst , i *goarch .PtrSize ))
*dst = (*dst )&^(((1 <<(min (bits , ptrBits )))-1 )<<j ) | (src << j )
}
const doubleCheck = false
if doubleCheck {
writeHeapBitsDoubleCheck (span , x , dataSize , src , src0 , i , j , bits , typ )
}
if doubleCheckHeapSetType {
doubleCheckHeapType (x , dataSize , typ , nil , span )
}
c .scanAlloc += scanSize
}
publicationBarrier ()
if writeBarrier .enabled {
gcmarknewobject (span , uintptr (x ))
} else {
span .freeIndexForScan = span .freeindex
}
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
gp := getg ()
if goexperiment .RuntimeSecret && gp .secret > 0 {
addSecret (x , size )
}
if gcBlackenEnabled != 0 && elemsize != 0 {
if assistG := getg ().m .curg ; assistG != nil {
assistG .gcAssistBytes -= int64 (elemsize - size )
}
}
if debug .malloc {
postMallocgcDebug (x , elemsize , typ )
}
return x
}
func mallocgcSmallNoScanSlowPath(size uintptr , typ *_type , needzero bool , spc spanClass , elemsize uintptr ) unsafe .Pointer {
if doubleCheckMalloc {
if gcphase == _GCmarktermination {
throw ("mallocgc called with gcphase == _GCmarktermination" )
}
}
var mp *m
lockRankMayQueueFinalizer ()
if debug .malloc {
if x := preMallocgcDebug (size , typ ); x != nil {
return x
}
}
if gcBlackenEnabled != 0 {
assistG := getg ()
if assistG .m .curg != nil {
assistG = assistG .m .curg
}
assistG .gcAssistBytes -= int64 (size )
if assistG .gcAssistBytes < 0 {
gcAssistAlloc (assistG )
}
}
mp = acquirem ()
if doubleCheckMalloc {
doubleCheckSmallNoScan (typ , mp )
}
mp .mallocing = 1
checkGCTrigger := false
c := getMCache (mp )
span := c .alloc [spc ]
var v gclinkptr
var x unsafe .Pointer
if runtimeFreegcEnabled && c .hasReusableNoscan (spc ) {
x = mallocgcSmallNoscanReuse (c , span , spc , elemsize , needzero )
mp .mallocing = 0
releasem (mp )
goto post
}
{
var nextFreeFastResult gclinkptr
if span .allocCache != 0 {
theBit := sys .TrailingZeros64 (span .allocCache )
result := span .freeindex + uint16 (theBit )
if result < span .nelems {
freeidx := result + 1
if !(freeidx %64 == 0 && freeidx != span .nelems ) {
span .allocCache >>= uint (theBit + 1 )
span .freeindex = freeidx
span .allocCount ++
nextFreeFastResult = gclinkptr (uintptr (result )*elemsize + span .base ())
}
}
}
v = nextFreeFastResult
if v == 0 {
v , span , checkGCTrigger = c .nextFree (spc )
}
x = unsafe .Pointer (v )
}
if needzero && span .needzero != 0 {
memclrNoHeapPointers (x , elemsize )
}
publicationBarrier ()
if writeBarrier .enabled {
gcmarknewobject (span , uintptr (x ))
} else {
span .freeIndexForScan = span .freeindex
}
c .nextSample -= int64 (elemsize )
if c .nextSample < 0 || MemProfileRate != c .memProfRate {
profilealloc (mp , x , elemsize )
}
mp .mallocing = 0
releasem (mp )
if checkGCTrigger {
if t := (gcTrigger {kind : gcTriggerHeap }); t .test () {
gcStart (t )
}
}
post :
gp := getg ()
if goexperiment .RuntimeSecret && gp .secret > 0 {
addSecret (x , size )
}
if gcBlackenEnabled != 0 && elemsize != 0 {
if assistG := getg ().m .curg ; assistG != nil {
assistG .gcAssistBytes -= int64 (elemsize - size )
}
}
if debug .malloc {
postMallocgcDebug (x , elemsize , typ )
}
return x
}
The pages are generated with Golds v0.8.7 . (GOOS=linux GOARCH=amd64)
Golds is a Go 101 project developed by Tapir Liu .
PR and bug reports are welcome and can be submitted to the issue list .
Please follow @zigo_101 (reachable from the left QR code) to get the latest news of Golds .