1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

void  OutOfMemoryPanic(size_t size);

void *SysAllocRaw(size_t size, int reqsize);
void  SysFreeRaw(void *ptr, size_t size);

void *AllocRaw4KB(int reqsize);
void *AllocRaw64KB(int reqsize);
void *LAlloc(size_t& size);
void  LFree(void *ptr);

struct Heap {
	enum { CACHE = 16 };

	static int Ksz(int k) {
		return k < 14 ? (k + 1) << 4 : k == 17 ? 576 : k == 16 ? 448 : k == 15 ? 368 : 288;
	}

	struct FreeLink {
		FreeLink *next;
	};

	struct Page {
		byte         klass;
		byte         active;
		Heap        *heap;
		FreeLink    *freelist;
		Page        *next;
		Page        *prev;

		void         LinkSelf()            { Dbl_Self(this); }
		void         Unlink()              { Dbl_Unlink(this); }
		void         Link(Page *lnk)       { Dbl_LinkAfter(this, lnk);  }

		void         Format(int k);

		byte *Begin()                      { return (byte *)this + sizeof(Page); }
		byte *End()                        { return (byte *)this + 4096; }
		int   Count()                      { return (int)(uintptr_t)(End() - Begin()) / Ksz(klass); }
	};

	struct Header;

	struct DLink {
		DLink       *next;
		DLink       *prev;

		void         LinkSelf()            { Dbl_Self(this); }
		void         Unlink()              { Dbl_Unlink(this); }
		void         Link(DLink *lnk)      { Dbl_LinkAfter(this, lnk);  }

		Header      *GetHeader()           { return (Header *)this - 1; }
	};

	struct Header {
		byte    free;
		byte    filler1, filler2, filler3;
		word    size;
		word    prev;
		Heap   *heap;
	#ifdef CPU_32
		dword   filler4;
	#endif

		DLink      *GetBlock()             { return (DLink *)(this + 1); }
		Header     *Next()                 { return (Header *)((byte *)this + size) + 1; }
		Header     *Prev()                 { return (Header *)((byte *)this - prev) - 1; }
	};

	struct BigHdr : DLink {
		size_t       size;
	};

	enum {
		NKLASS = 18,
		LBINS = 113,
		LARGEHDRSZ = 24,
		MAXBLOCK = 65536 - 2 * sizeof(Header) - LARGEHDRSZ,
		BIGHDRSZ = 56,
	};
	static StaticMutex mutex;

	Page      work[NKLASS][1];
	Page      full[NKLASS][1];
	Page     *empty[NKLASS];
	FreeLink *cache[NKLASS];
	int       cachen[NKLASS];

	bool      initialized;

	static word  BinSz[LBINS];
	static byte  SzBin[MAXBLOCK / 8 + 1];
	static byte  BlBin[MAXBLOCK / 8 + 1];

	DLink  large[1];
	int    lcount;
	DLink  freebin[LBINS][1];
	static DLink lempty[1];

	FreeLink *remote_free;

	static DLink big[1];
	static Heap  aux;

#ifdef HEAPDBG
	static void  DbgFreeFill(void *ptr, size_t size);
	static void  DbgFreeCheck(void *ptr, size_t size);
	static void  DbgFreeFillK(void *ptr, int k);
	static void *DbgFreeCheckK(void *p, int k);
#else
	static void  DbgFreeFill(void *ptr, size_t size) {}
	static void  DbgFreeCheck(void *ptr, size_t size) {}
	static void  DbgFreeFillK(void *ptr, int k) {}
	static void *DbgFreeCheckK(void *p, int k) { return p; }
#endif

#ifdef flagHEAPSTAT
	static void  Stat(size_t sz);
#else
	static void  Stat(size_t sz) {}
#endif

	void  FreeRemoteRaw();
	void  FreeRemote();

	void  Init();

	static int   CheckPageFree(FreeLink *l, int k);
	void  Check();
	static void  Assert(bool b);
	static void  DblCheck(Page *p);
	static void  AssertLeaks(bool b);

	Page *WorkPage(int k);
	void *AllocK(int k);
	void  FreeK(void *ptr, Page *page, int k);
	void *Allok(int k);
	void  Freek(void *ptr, int k);
	void  FreeDirect(void *ptr);
	void  MoveLarge(Heap *dest, DLink *l);
	void  MoveToEmpty(DLink *l, Header *bh);

	static void GlobalLInit();
	static int  SizeToBin(int n) { return SzBin[(n - 1) >> 3]; }

	void   LinkFree(DLink *b, int size);
	DLink *AddChunk(int reqsize);
	void  *DivideBlock(DLink *b, int size, int ii);
	void  *TryLAlloc(int ii, size_t size);
	void  *LAlloc(size_t& size);
	void   LFree(void *ptr);
	void   Make(MemoryProfile& f);

	void RemoteFree(void *ptr);
	void Shutdown();
	static void AuxFinalCheck();

	void *Alloc(size_t sz);
	void *AllocSz(size_t& sz);
	void  Free(void *ptr);
	void *Alloc32();
	void  Free32(void *ptr);
	void *Alloc48();
	void  Free48(void *ptr);
};

extern thread__ Heap heap;