1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : *
3 : * ***** BEGIN LICENSE BLOCK *****
4 : * Version: MPL 1.1/GPL 2.0/LGPL 2.1
5 : *
6 : * The contents of this file are subject to the Mozilla Public License Version
7 : * 1.1 (the "License"); you may not use this file except in compliance with
8 : * the License. You may obtain a copy of the License at
9 : * http://www.mozilla.org/MPL/
10 : *
11 : * Software distributed under the License is distributed on an "AS IS" basis,
12 : * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
13 : * for the specific language governing rights and limitations under the
14 : * License.
15 : *
16 : * The Original Code is Mozilla Communicator client code, released
17 : * March 31, 1998.
18 : *
19 : * The Initial Developer of the Original Code is
20 : * Netscape Communications Corporation.
21 : * Portions created by the Initial Developer are Copyright (C) 1998
22 : * the Initial Developer. All Rights Reserved.
23 : *
24 : * Contributor(s):
25 : *
26 : * Alternatively, the contents of this file may be used under the terms of
27 : * either of the GNU General Public License Version 2 or later (the "GPL"),
28 : * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
29 : * in which case the provisions of the GPL or the LGPL are applicable instead
30 : * of those above. If you wish to allow use of your version of this file only
31 : * under the terms of either the GPL or the LGPL, and not to allow others to
32 : * use your version of this file under the terms of the MPL, indicate your
33 : * decision by deleting the provisions above and replace them with the notice
34 : * and other provisions required by the GPL or the LGPL. If you do not delete
35 : * the provisions above, a recipient may use your version of this file under
36 : * the terms of any one of the MPL, the GPL or the LGPL.
37 : *
38 : * ***** END LICENSE BLOCK ***** */
39 :
40 : #ifndef jsgc_h___
41 : #define jsgc_h___
42 :
43 : /*
44 : * JS Garbage Collector.
45 : */
46 : #include <setjmp.h>
47 :
48 : #include "mozilla/Util.h"
49 :
50 : #include "jsalloc.h"
51 : #include "jstypes.h"
52 : #include "jsprvtd.h"
53 : #include "jspubtd.h"
54 : #include "jslock.h"
55 : #include "jsutil.h"
56 : #include "jsversion.h"
57 : #include "jscell.h"
58 :
59 : #include "ds/BitArray.h"
60 : #include "gc/Statistics.h"
61 : #include "js/HashTable.h"
62 : #include "js/Vector.h"
63 : #include "js/TemplateLib.h"
64 :
65 : struct JSCompartment;
66 :
67 : extern "C" void
68 : js_TraceXML(JSTracer *trc, JSXML* thing);
69 :
70 : #if JS_STACK_GROWTH_DIRECTION > 0
71 : # define JS_CHECK_STACK_SIZE(limit, lval) ((uintptr_t)(lval) < limit)
72 : #else
73 : # define JS_CHECK_STACK_SIZE(limit, lval) ((uintptr_t)(lval) > limit)
74 : #endif
75 :
76 : namespace js {
77 :
78 : class GCHelperThread;
79 : struct Shape;
80 :
81 : namespace gc {
82 :
83 : enum State {
84 : NO_INCREMENTAL,
85 : MARK_ROOTS,
86 : MARK,
87 : SWEEP,
88 : INVALID
89 : };
90 :
91 : struct Arena;
92 :
93 : /*
94 : * This must be an upper bound, but we do not need the least upper bound, so
95 : * we just exclude non-background objects.
96 : */
97 : const size_t MAX_BACKGROUND_FINALIZE_KINDS = FINALIZE_LIMIT - FINALIZE_OBJECT_LIMIT / 2;
98 :
99 : /*
100 : * Page size is 4096 by default, except for SPARC, where it is 8192.
101 : * Note: Do not use JS_CPU_SPARC here, this header is used outside JS.
102 : * Bug 692267: Move page size definition to gc/Memory.h and include it
103 : * directly once jsgc.h is no longer an installed header.
104 : */
105 : #if defined(SOLARIS) && (defined(__sparc) || defined(__sparcv9))
106 : const size_t PageShift = 13;
107 : #else
108 : const size_t PageShift = 12;
109 : #endif
110 : const size_t PageSize = size_t(1) << PageShift;
111 :
112 : const size_t ChunkShift = 20;
113 : const size_t ChunkSize = size_t(1) << ChunkShift;
114 : const size_t ChunkMask = ChunkSize - 1;
115 :
116 : const size_t ArenaShift = PageShift;
117 : const size_t ArenaSize = PageSize;
118 : const size_t ArenaMask = ArenaSize - 1;
119 :
120 : /*
121 : * This is the maximum number of arenas we allow in the FreeCommitted state
122 : * before we trigger a GC_SHRINK to release free arenas to the OS.
123 : */
124 : const static uint32_t FreeCommittedArenasThreshold = (32 << 20) / ArenaSize;
125 :
126 : /*
127 : * The mark bitmap has one bit per each GC cell. For multi-cell GC things this
128 : * wastes space but allows to avoid expensive devisions by thing's size when
129 : * accessing the bitmap. In addition this allows to use some bits for colored
130 : * marking during the cycle GC.
131 : */
132 : const size_t ArenaCellCount = size_t(1) << (ArenaShift - Cell::CellShift);
133 : const size_t ArenaBitmapBits = ArenaCellCount;
134 : const size_t ArenaBitmapBytes = ArenaBitmapBits / 8;
135 : const size_t ArenaBitmapWords = ArenaBitmapBits / JS_BITS_PER_WORD;
136 :
137 : /*
138 : * A FreeSpan represents a contiguous sequence of free cells in an Arena.
139 : * |first| is the address of the first free cell in the span. |last| is the
140 : * address of the last free cell in the span. This last cell holds a FreeSpan
141 : * data structure for the next span unless this is the last span on the list
142 : * of spans in the arena. For this last span |last| points to the last byte of
143 : * the last thing in the arena and no linkage is stored there, so
144 : * |last| == arenaStart + ArenaSize - 1. If the space at the arena end is
145 : * fully used this last span is empty and |first| == |last + 1|.
146 : *
147 : * Thus |first| < |last| implies that we have either the last span with at least
148 : * one element or that the span is not the last and contains at least 2
149 : * elements. In both cases to allocate a thing from this span we need simply
150 : * to increment |first| by the allocation size.
151 : *
152 : * |first| == |last| implies that we have a one element span that records the
153 : * next span. So to allocate from it we need to update the span list head
154 : * with a copy of the span stored at |last| address so the following
155 : * allocations will use that span.
156 : *
157 : * |first| > |last| implies that we have an empty last span and the arena is
158 : * fully used.
159 : *
160 : * Also only for the last span (|last| & 1)! = 0 as all allocation sizes are
161 : * multiples of Cell::CellSize.
162 : */
163 : struct FreeSpan {
164 : uintptr_t first;
165 : uintptr_t last;
166 :
167 : public:
168 4049092 : FreeSpan() {}
169 :
170 23687219 : FreeSpan(uintptr_t first, uintptr_t last)
171 23687219 : : first(first), last(last) {
172 23687219 : checkSpan();
173 23687216 : }
174 :
175 : /*
176 : * To minimize the size of the arena header the first span is encoded
177 : * there as offsets from the arena start.
178 : */
179 2674283 : static size_t encodeOffsets(size_t firstOffset, size_t lastOffset) {
180 : /* Check that we can pack the offsets into uint16. */
181 : JS_STATIC_ASSERT(ArenaShift < 16);
182 2674283 : JS_ASSERT(firstOffset <= ArenaSize);
183 2674282 : JS_ASSERT(lastOffset < ArenaSize);
184 2674282 : JS_ASSERT(firstOffset <= ((lastOffset + 1) & ~size_t(1)));
185 2674282 : return firstOffset | (lastOffset << 16);
186 : }
187 :
188 : /*
189 : * Encoded offsets for a full arena when its first span is the last one
190 : * and empty.
191 : */
192 : static const size_t FullArenaOffsets = ArenaSize | ((ArenaSize - 1) << 16);
193 :
194 23687236 : static FreeSpan decodeOffsets(uintptr_t arenaAddr, size_t offsets) {
195 23687236 : JS_ASSERT(!(arenaAddr & ArenaMask));
196 :
197 23687236 : size_t firstOffset = offsets & 0xFFFF;
198 23687236 : size_t lastOffset = offsets >> 16;
199 23687236 : JS_ASSERT(firstOffset <= ArenaSize);
200 23687236 : JS_ASSERT(lastOffset < ArenaSize);
201 :
202 : /*
203 : * We must not use | when calculating first as firstOffset is
204 : * ArenaMask + 1 for the empty span.
205 : */
206 23687236 : return FreeSpan(arenaAddr + firstOffset, arenaAddr | lastOffset);
207 : }
208 :
209 1502845 : void initAsEmpty(uintptr_t arenaAddr = 0) {
210 1502845 : JS_ASSERT(!(arenaAddr & ArenaMask));
211 1502845 : first = arenaAddr + ArenaSize;
212 1502845 : last = arenaAddr | (ArenaSize - 1);
213 1502845 : JS_ASSERT(isEmpty());
214 1502845 : }
215 :
216 32417812 : bool isEmpty() const {
217 32417812 : checkSpan();
218 32417812 : return first > last;
219 : }
220 :
221 14160861 : bool hasNext() const {
222 14160861 : checkSpan();
223 14160861 : return !(last & uintptr_t(1));
224 : }
225 :
226 13598294 : const FreeSpan *nextSpan() const {
227 13598294 : JS_ASSERT(hasNext());
228 13598294 : return reinterpret_cast<FreeSpan *>(last);
229 : }
230 :
231 2397087 : FreeSpan *nextSpanUnchecked(size_t thingSize) const {
232 : #ifdef DEBUG
233 2397087 : uintptr_t lastOffset = last & ArenaMask;
234 2397087 : JS_ASSERT(!(lastOffset & 1));
235 2397087 : JS_ASSERT((ArenaSize - lastOffset) % thingSize == 0);
236 : #endif
237 2397087 : return reinterpret_cast<FreeSpan *>(last);
238 : }
239 :
240 562421111 : uintptr_t arenaAddressUnchecked() const {
241 562421111 : return last & ~ArenaMask;
242 : }
243 :
244 8100228 : uintptr_t arenaAddress() const {
245 8100228 : checkSpan();
246 8100228 : return arenaAddressUnchecked();
247 : }
248 :
249 1224592 : ArenaHeader *arenaHeader() const {
250 1224592 : return reinterpret_cast<ArenaHeader *>(arenaAddress());
251 : }
252 :
253 969496 : bool isSameNonEmptySpan(const FreeSpan *another) const {
254 969496 : JS_ASSERT(!isEmpty());
255 969496 : JS_ASSERT(!another->isEmpty());
256 969496 : return first == another->first && last == another->last;
257 : }
258 :
259 2589169 : bool isWithinArena(uintptr_t arenaAddr) const {
260 2589169 : JS_ASSERT(!(arenaAddr & ArenaMask));
261 :
262 : /* Return true for the last empty span as well. */
263 2589169 : return arenaAddress() == arenaAddr;
264 : }
265 :
266 2589169 : size_t encodeAsOffsets() const {
267 : /*
268 : * We must use first - arenaAddress(), not first & ArenaMask as
269 : * first == ArenaMask + 1 for an empty span.
270 : */
271 2589169 : uintptr_t arenaAddr = arenaAddress();
272 2589169 : return encodeOffsets(first - arenaAddr, last & ArenaMask);
273 : }
274 :
275 : /* See comments before FreeSpan for details. */
276 214032028 : JS_ALWAYS_INLINE void *allocate(size_t thingSize) {
277 214032028 : JS_ASSERT(thingSize % Cell::CellSize == 0);
278 214032028 : checkSpan();
279 214032028 : uintptr_t thing = first;
280 214032028 : if (thing < last) {
281 : /* Bump-allocate from the current span. */
282 211528816 : first = thing + thingSize;
283 2503212 : } else if (JS_LIKELY(thing == last)) {
284 : /*
285 : * Move to the next span. We use JS_LIKELY as without PGO
286 : * compilers mis-predict == here as unlikely to succeed.
287 : */
288 493116 : *this = *reinterpret_cast<FreeSpan *>(thing);
289 : } else {
290 2010096 : return NULL;
291 : }
292 212021932 : checkSpan();
293 212021932 : return reinterpret_cast<void *>(thing);
294 : }
295 :
296 : /* A version of allocate when we know that the span is not empty. */
297 85138 : JS_ALWAYS_INLINE void *infallibleAllocate(size_t thingSize) {
298 85138 : JS_ASSERT(thingSize % Cell::CellSize == 0);
299 85138 : checkSpan();
300 85138 : uintptr_t thing = first;
301 85138 : if (thing < last) {
302 49619 : first = thing + thingSize;
303 : } else {
304 35519 : JS_ASSERT(thing == last);
305 35519 : *this = *reinterpret_cast<FreeSpan *>(thing);
306 : }
307 85138 : checkSpan();
308 85138 : return reinterpret_cast<void *>(thing);
309 : }
310 :
311 : /*
312 : * Allocate from a newly allocated arena. We do not move the free list
313 : * from the arena. Rather we set the arena up as fully used during the
314 : * initialization so to allocate we simply return the first thing in the
315 : * arena and set the free list to point to the second.
316 : */
317 1683327 : JS_ALWAYS_INLINE void *allocateFromNewArena(uintptr_t arenaAddr, size_t firstThingOffset,
318 : size_t thingSize) {
319 1683327 : JS_ASSERT(!(arenaAddr & ArenaMask));
320 1683327 : uintptr_t thing = arenaAddr | firstThingOffset;
321 1683327 : first = thing + thingSize;
322 1683327 : last = arenaAddr | ArenaMask;
323 1683327 : checkSpan();
324 1683327 : return reinterpret_cast<void *>(thing);
325 : }
326 :
327 513450769 : void checkSpan() const {
328 : #ifdef DEBUG
329 : /* We do not allow spans at the end of the address space. */
330 513450769 : JS_ASSERT(last != uintptr_t(-1));
331 513450164 : JS_ASSERT(first);
332 513450164 : JS_ASSERT(last);
333 513450164 : JS_ASSERT(first - 1 <= last);
334 513450164 : uintptr_t arenaAddr = arenaAddressUnchecked();
335 513468657 : if (last & 1) {
336 : /* The span is the last. */
337 472565072 : JS_ASSERT((last & ArenaMask) == ArenaMask);
338 :
339 472569426 : if (first - 1 == last) {
340 : /* The span is last and empty. The above start != 0 check
341 : * implies that we are not at the end of the address space.
342 : */
343 29325505 : return;
344 : }
345 443243921 : size_t spanLength = last - first + 1;
346 443243921 : JS_ASSERT(spanLength % Cell::CellSize == 0);
347 :
348 : /* Start and end must belong to the same arena. */
349 443243888 : JS_ASSERT((first & ~ArenaMask) == arenaAddr);
350 443243888 : return;
351 : }
352 :
353 : /* The span is not the last and we have more spans to follow. */
354 40903585 : JS_ASSERT(first <= last);
355 40903521 : size_t spanLengthWithoutOneThing = last - first;
356 40903521 : JS_ASSERT(spanLengthWithoutOneThing % Cell::CellSize == 0);
357 :
358 40903521 : JS_ASSERT((first & ~ArenaMask) == arenaAddr);
359 :
360 : /*
361 : * If there is not enough space before the arena end to allocate one
362 : * more thing, then the span must be marked as the last one to avoid
363 : * storing useless empty span reference.
364 : */
365 40903521 : size_t beforeTail = ArenaSize - (last & ArenaMask);
366 40903521 : JS_ASSERT(beforeTail >= sizeof(FreeSpan) + Cell::CellSize);
367 :
368 40903521 : FreeSpan *next = reinterpret_cast<FreeSpan *>(last);
369 :
370 : /*
371 : * The GC things on the list of free spans come from one arena
372 : * and the spans are linked in ascending address order with
373 : * at least one non-free thing between spans.
374 : */
375 40903521 : JS_ASSERT(last < next->first);
376 40903521 : JS_ASSERT(arenaAddr == next->arenaAddressUnchecked());
377 :
378 40903758 : if (next->first > next->last) {
379 : /*
380 : * The next span is the empty span that terminates the list for
381 : * arenas that do not have any free things at the end.
382 : */
383 5330768 : JS_ASSERT(next->first - 1 == next->last);
384 5330767 : JS_ASSERT(arenaAddr + ArenaSize == next->first);
385 : }
386 : #endif
387 : }
388 :
389 : };
390 :
391 : /* Every arena has a header. */
392 : struct ArenaHeader {
393 : friend struct FreeLists;
394 :
395 : JSCompartment *compartment;
396 :
397 : /*
398 : * ArenaHeader::next has two purposes: when unallocated, it points to the
399 : * next available Arena's header. When allocated, it points to the next
400 : * arena of the same size class and compartment.
401 : */
402 : ArenaHeader *next;
403 :
404 : private:
405 : /*
406 : * The first span of free things in the arena. We encode it as the start
407 : * and end offsets within the arena, not as FreeSpan structure, to
408 : * minimize the header size.
409 : */
410 : size_t firstFreeSpanOffsets;
411 :
412 : /*
413 : * One of AllocKind constants or FINALIZE_LIMIT when the arena does not
414 : * contain any GC things and is on the list of empty arenas in the GC
415 : * chunk. The latter allows to quickly check if the arena is allocated
416 : * during the conservative GC scanning without searching the arena in the
417 : * list.
418 : */
419 : size_t allocKind : 8;
420 :
421 : /*
422 : * When recursive marking uses too much stack the marking is delayed and
423 : * the corresponding arenas are put into a stack using the following field
424 : * as a linkage. To distinguish the bottom of the stack from the arenas
425 : * not present in the stack we use an extra flag to tag arenas on the
426 : * stack.
427 : *
428 : * Delayed marking is also used for arenas that we allocate into during an
429 : * incremental GC. In this case, we intend to mark all the objects in the
430 : * arena, and it's faster to do this marking in bulk.
431 : *
432 : * To minimize the ArenaHeader size we record the next delayed marking
433 : * linkage as arenaAddress() >> ArenaShift and pack it with the allocKind
434 : * field and hasDelayedMarking flag. We use 8 bits for the allocKind, not
435 : * ArenaShift - 1, so the compiler can use byte-level memory instructions
436 : * to access it.
437 : */
438 : public:
439 : size_t hasDelayedMarking : 1;
440 : size_t allocatedDuringIncremental : 1;
441 : size_t markOverflow : 1;
442 : size_t nextDelayedMarking : JS_BITS_PER_WORD - 8 - 1 - 1 - 1;
443 :
444 : static void staticAsserts() {
445 : /* We must be able to fit the allockind into uint8_t. */
446 : JS_STATIC_ASSERT(FINALIZE_LIMIT <= 255);
447 :
448 : /*
449 : * nextDelayedMarkingpacking assumes that ArenaShift has enough bits
450 : * to cover allocKind and hasDelayedMarking.
451 : */
452 : JS_STATIC_ASSERT(ArenaShift >= 8 + 1 + 1 + 1);
453 : }
454 :
455 : inline uintptr_t address() const;
456 : inline Chunk *chunk() const;
457 :
458 811666512 : bool allocated() const {
459 811666512 : JS_ASSERT(allocKind <= size_t(FINALIZE_LIMIT));
460 811666512 : return allocKind < size_t(FINALIZE_LIMIT);
461 : }
462 :
463 1683327 : void init(JSCompartment *comp, AllocKind kind) {
464 1683327 : JS_ASSERT(!allocated());
465 1683327 : JS_ASSERT(!markOverflow);
466 1683327 : JS_ASSERT(!allocatedDuringIncremental);
467 1683327 : JS_ASSERT(!hasDelayedMarking);
468 1683327 : compartment = comp;
469 :
470 : JS_STATIC_ASSERT(FINALIZE_LIMIT <= 255);
471 1683327 : allocKind = size_t(kind);
472 :
473 : /* See comments in FreeSpan::allocateFromNewArena. */
474 1683327 : firstFreeSpanOffsets = FreeSpan::FullArenaOffsets;
475 1683327 : }
476 :
477 11887059 : void setAsNotAllocated() {
478 11887059 : allocKind = size_t(FINALIZE_LIMIT);
479 11887059 : markOverflow = 0;
480 11887059 : allocatedDuringIncremental = 0;
481 11887059 : hasDelayedMarking = 0;
482 11887059 : nextDelayedMarking = 0;
483 11887059 : }
484 :
485 31190911 : uintptr_t arenaAddress() const {
486 31190911 : return address();
487 : }
488 :
489 3004483 : Arena *getArena() {
490 3004483 : return reinterpret_cast<Arena *>(arenaAddress());
491 : }
492 :
493 493583740 : AllocKind getAllocKind() const {
494 493583740 : JS_ASSERT(allocated());
495 493583474 : return AllocKind(allocKind);
496 : }
497 :
498 : inline size_t getThingSize() const;
499 :
500 15174922 : bool hasFreeThings() const {
501 15174922 : return firstFreeSpanOffsets != FreeSpan::FullArenaOffsets;
502 : }
503 :
504 : inline bool isEmpty() const;
505 :
506 228511 : void setAsFullyUsed() {
507 228511 : firstFreeSpanOffsets = FreeSpan::FullArenaOffsets;
508 228511 : }
509 :
510 12401776 : FreeSpan getFirstFreeSpan() const {
511 : #ifdef DEBUG
512 12401776 : checkSynchronizedWithFreeList();
513 : #endif
514 12401679 : return FreeSpan::decodeOffsets(arenaAddress(), firstFreeSpanOffsets);
515 : }
516 :
517 2589169 : void setFirstFreeSpan(const FreeSpan *span) {
518 2589169 : JS_ASSERT(span->isWithinArena(arenaAddress()));
519 2589169 : firstFreeSpanOffsets = span->encodeAsOffsets();
520 2589169 : }
521 :
522 : #ifdef DEBUG
523 : void checkSynchronizedWithFreeList() const;
524 : #endif
525 :
526 : inline ArenaHeader *getNextDelayedMarking() const;
527 : inline void setNextDelayedMarking(ArenaHeader *aheader);
528 : };
529 :
530 : struct Arena {
531 : /*
532 : * Layout of an arena:
533 : * An arena is 4K in size and 4K-aligned. It starts with the ArenaHeader
534 : * descriptor followed by some pad bytes. The remainder of the arena is
535 : * filled with the array of T things. The pad bytes ensure that the thing
536 : * array ends exactly at the end of the arena.
537 : *
538 : * +-------------+-----+----+----+-----+----+
539 : * | ArenaHeader | pad | T0 | T1 | ... | Tn |
540 : * +-------------+-----+----+----+-----+----+
541 : *
542 : * <----------------------------------------> = ArenaSize bytes
543 : * <-------------------> = first thing offset
544 : */
545 : ArenaHeader aheader;
546 : uint8_t data[ArenaSize - sizeof(ArenaHeader)];
547 :
548 : private:
549 : static JS_FRIEND_DATA(const uint32_t) ThingSizes[];
550 : static JS_FRIEND_DATA(const uint32_t) FirstThingOffsets[];
551 :
552 : public:
553 : static void staticAsserts();
554 :
555 554979090 : static size_t thingSize(AllocKind kind) {
556 554979090 : return ThingSizes[kind];
557 : }
558 :
559 20091857 : static size_t firstThingOffset(AllocKind kind) {
560 20091857 : return FirstThingOffsets[kind];
561 : }
562 :
563 4025285 : static size_t thingsPerArena(size_t thingSize) {
564 4025285 : JS_ASSERT(thingSize % Cell::CellSize == 0);
565 :
566 : /* We should be able to fit FreeSpan in any GC thing. */
567 4025285 : JS_ASSERT(thingSize >= sizeof(FreeSpan));
568 :
569 4025285 : return (ArenaSize - sizeof(ArenaHeader)) / thingSize;
570 : }
571 :
572 : static size_t thingsSpan(size_t thingSize) {
573 : return thingsPerArena(thingSize) * thingSize;
574 : }
575 :
576 271476876 : static bool isAligned(uintptr_t thing, size_t thingSize) {
577 : /* Things ends at the arena end. */
578 271476876 : uintptr_t tailOffset = (ArenaSize - thing) & ArenaMask;
579 271476876 : return tailOffset % thingSize == 0;
580 : }
581 :
582 10507103 : uintptr_t address() const {
583 10507103 : return aheader.address();
584 : }
585 :
586 6724665 : uintptr_t thingsStart(AllocKind thingKind) {
587 6724665 : return address() | firstThingOffset(thingKind);
588 : }
589 :
590 3782438 : uintptr_t thingsEnd() {
591 3782438 : return address() + ArenaSize;
592 : }
593 :
594 : template <typename T>
595 : bool finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize);
596 : };
597 :
598 : /* The chunk header (located at the end of the chunk to preserve arena alignment). */
599 : struct ChunkInfo {
600 : Chunk *next;
601 : Chunk **prevp;
602 :
603 : /* Free arenas are linked together with aheader.next. */
604 : ArenaHeader *freeArenasHead;
605 :
606 : /*
607 : * Decommitted arenas are tracked by a bitmap in the chunk header. We use
608 : * this offset to start our search iteration close to a decommitted arena
609 : * that we can allocate.
610 : */
611 : uint32_t lastDecommittedArenaOffset;
612 :
613 : /* Number of free arenas, either committed or decommitted. */
614 : uint32_t numArenasFree;
615 :
616 : /* Number of free, committed arenas. */
617 : uint32_t numArenasFreeCommitted;
618 :
619 : /* Number of GC cycles this chunk has survived. */
620 : uint32_t age;
621 : };
622 :
623 : /*
624 : * Calculating ArenasPerChunk:
625 : *
626 : * In order to figure out how many Arenas will fit in a chunk, we need to know
627 : * how much extra space is available after we allocate the header data. This
628 : * is a problem because the header size depends on the number of arenas in the
629 : * chunk. The two dependent fields are bitmap and decommittedArenas.
630 : *
631 : * For the mark bitmap, we know that each arena will use a fixed number of full
632 : * bytes: ArenaBitmapBytes. The full size of the header data is this number
633 : * multiplied by the eventual number of arenas we have in the header. We,
634 : * conceptually, distribute this header data among the individual arenas and do
635 : * not include it in the header. This way we do not have to worry about its
636 : * variable size: it gets attached to the variable number we are computing.
637 : *
638 : * For the decommitted arena bitmap, we only have 1 bit per arena, so this
639 : * technique will not work. Instead, we observe that we do not have enough
640 : * header info to fill 8 full arenas: it is currently 4 on 64bit, less on
641 : * 32bit. Thus, with current numbers, we need 64 bytes for decommittedArenas.
642 : * This will not become 63 bytes unless we double the data required in the
643 : * header. Therefore, we just compute the number of bytes required to track
644 : * every possible arena and do not worry about slop bits, since there are too
645 : * few to usefully allocate.
646 : *
647 : * To actually compute the number of arenas we can allocate in a chunk, we
648 : * divide the amount of available space less the header info (not including
649 : * the mark bitmap which is distributed into the arena size) by the size of
650 : * the arena (with the mark bitmap bytes it uses).
651 : */
652 : const size_t BytesPerArenaWithHeader = ArenaSize + ArenaBitmapBytes;
653 : const size_t ChunkDecommitBitmapBytes = ChunkSize / ArenaSize / JS_BITS_PER_BYTE;
654 : const size_t ChunkBytesAvailable = ChunkSize - sizeof(ChunkInfo) - ChunkDecommitBitmapBytes;
655 : const size_t ArenasPerChunk = ChunkBytesAvailable / BytesPerArenaWithHeader;
656 :
657 : /* A chunk bitmap contains enough mark bits for all the cells in a chunk. */
658 : struct ChunkBitmap {
659 : uintptr_t bitmap[ArenaBitmapWords * ArenasPerChunk];
660 :
661 : JS_ALWAYS_INLINE void getMarkWordAndMask(const Cell *cell, uint32_t color,
662 : uintptr_t **wordp, uintptr_t *maskp);
663 :
664 642428391 : JS_ALWAYS_INLINE bool isMarked(const Cell *cell, uint32_t color) {
665 : uintptr_t *word, mask;
666 642428391 : getMarkWordAndMask(cell, color, &word, &mask);
667 642428391 : return *word & mask;
668 : }
669 :
670 366717501 : JS_ALWAYS_INLINE bool markIfUnmarked(const Cell *cell, uint32_t color) {
671 : uintptr_t *word, mask;
672 366717501 : getMarkWordAndMask(cell, BLACK, &word, &mask);
673 366717501 : if (*word & mask)
674 117678446 : return false;
675 249039055 : *word |= mask;
676 249039055 : if (color != BLACK) {
677 : /*
678 : * We use getMarkWordAndMask to recalculate both mask and word as
679 : * doing just mask << color may overflow the mask.
680 : */
681 0 : getMarkWordAndMask(cell, color, &word, &mask);
682 0 : if (*word & mask)
683 0 : return false;
684 0 : *word |= mask;
685 : }
686 249039055 : return true;
687 : }
688 :
689 : JS_ALWAYS_INLINE void unmark(const Cell *cell, uint32_t color) {
690 : uintptr_t *word, mask;
691 : getMarkWordAndMask(cell, color, &word, &mask);
692 : *word &= ~mask;
693 : }
694 :
695 123883 : void clear() {
696 123883 : PodArrayZero(bitmap);
697 123883 : }
698 :
699 : #ifdef DEBUG
700 2248 : bool noBitsSet(ArenaHeader *aheader) {
701 : /*
702 : * We assume that the part of the bitmap corresponding to the arena
703 : * has the exact number of words so we do not need to deal with a word
704 : * that covers bits from two arenas.
705 : */
706 : JS_STATIC_ASSERT(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD);
707 :
708 : uintptr_t *word, unused;
709 2248 : getMarkWordAndMask(reinterpret_cast<Cell *>(aheader->address()), BLACK, &word, &unused);
710 38216 : for (size_t i = 0; i != ArenaBitmapWords; i++) {
711 35968 : if (word[i])
712 0 : return false;
713 : }
714 2248 : return true;
715 : }
716 : #endif
717 : };
718 :
719 : JS_STATIC_ASSERT(ArenaBitmapBytes * ArenasPerChunk == sizeof(ChunkBitmap));
720 :
721 : typedef BitArray<ArenasPerChunk> PerArenaBitmap;
722 :
723 : const size_t ChunkPadSize = ChunkSize
724 : - (sizeof(Arena) * ArenasPerChunk)
725 : - sizeof(ChunkBitmap)
726 : - sizeof(PerArenaBitmap)
727 : - sizeof(ChunkInfo);
728 : JS_STATIC_ASSERT(ChunkPadSize < BytesPerArenaWithHeader);
729 :
730 : /*
731 : * Chunks contain arenas and associated data structures (mark bitmap, delayed
732 : * marking state).
733 : */
734 : struct Chunk {
735 : Arena arenas[ArenasPerChunk];
736 :
737 : /* Pad to full size to ensure cache alignment of ChunkInfo. */
738 : uint8_t padding[ChunkPadSize];
739 :
740 : ChunkBitmap bitmap;
741 : PerArenaBitmap decommittedArenas;
742 : ChunkInfo info;
743 :
744 208257559 : static Chunk *fromAddress(uintptr_t addr) {
745 208257559 : addr &= ~ChunkMask;
746 208257559 : return reinterpret_cast<Chunk *>(addr);
747 : }
748 :
749 -1 : static bool withinArenasRange(uintptr_t addr) {
750 -1 : uintptr_t offset = addr & ChunkMask;
751 -1 : return offset < ArenasPerChunk * ArenaSize;
752 : }
753 :
754 11555918 : static size_t arenaIndex(uintptr_t addr) {
755 11555918 : JS_ASSERT(withinArenasRange(addr));
756 11555918 : return (addr & ChunkMask) >> ArenaShift;
757 : }
758 :
759 : uintptr_t address() const {
760 : uintptr_t addr = reinterpret_cast<uintptr_t>(this);
761 : JS_ASSERT(!(addr & ChunkMask));
762 : return addr;
763 : }
764 :
765 1765950 : bool unused() const {
766 1765950 : return info.numArenasFree == ArenasPerChunk;
767 : }
768 :
769 3418436 : bool hasAvailableArenas() const {
770 3418436 : return info.numArenasFree != 0;
771 : }
772 :
773 : inline void addToAvailableList(JSCompartment *compartment);
774 : inline void insertToAvailableList(Chunk **insertPoint);
775 : inline void removeFromAvailableList();
776 :
777 : ArenaHeader *allocateArena(JSCompartment *comp, AllocKind kind);
778 :
779 : void releaseArena(ArenaHeader *aheader);
780 :
781 : static Chunk *allocate(JSRuntime *rt);
782 :
783 : /* Must be called with the GC lock taken. */
784 : static inline void release(JSRuntime *rt, Chunk *chunk);
785 : static inline void releaseList(JSRuntime *rt, Chunk *chunkListHead);
786 :
787 : /* Must be called with the GC lock taken. */
788 : inline void prepareToBeFreed(JSRuntime *rt);
789 :
790 : /*
791 : * Assuming that the info.prevp points to the next field of the previous
792 : * chunk in a doubly-linked list, get that chunk.
793 : */
794 0 : Chunk *getPrevious() {
795 0 : JS_ASSERT(info.prevp);
796 0 : return fromPointerToNext(info.prevp);
797 : }
798 :
799 : /* Get the chunk from a pointer to its info.next field. */
800 0 : static Chunk *fromPointerToNext(Chunk **nextFieldPtr) {
801 0 : uintptr_t addr = reinterpret_cast<uintptr_t>(nextFieldPtr);
802 0 : JS_ASSERT((addr & ChunkMask) == offsetof(Chunk, info.next));
803 0 : return reinterpret_cast<Chunk *>(addr - offsetof(Chunk, info.next));
804 : }
805 :
806 : private:
807 : inline void init();
808 :
809 : /* Search for a decommitted arena to allocate. */
810 : unsigned findDecommittedArenaOffset();
811 : ArenaHeader* fetchNextDecommittedArena();
812 :
813 : public:
814 : /* Unlink and return the freeArenasHead. */
815 : inline ArenaHeader* fetchNextFreeArena(JSRuntime *rt);
816 :
817 : inline void addArenaToFreeList(JSRuntime *rt, ArenaHeader *aheader);
818 : };
819 :
820 : JS_STATIC_ASSERT(sizeof(Chunk) == ChunkSize);
821 :
822 : class ChunkPool {
823 : Chunk *emptyChunkListHead;
824 : size_t emptyCount;
825 :
826 : public:
827 18761 : ChunkPool()
828 : : emptyChunkListHead(NULL),
829 18761 : emptyCount(0) { }
830 :
831 0 : size_t getEmptyCount() const {
832 0 : return emptyCount;
833 : }
834 :
835 : inline bool wantBackgroundAllocation(JSRuntime *rt) const;
836 :
837 : /* Must be called with the GC lock taken. */
838 : inline Chunk *get(JSRuntime *rt);
839 :
840 : /* Must be called either during the GC or with the GC lock taken. */
841 : inline void put(Chunk *chunk);
842 :
843 : /*
844 : * Return the list of chunks that can be released outside the GC lock.
845 : * Must be called either during the GC or with the GC lock taken.
846 : */
847 : Chunk *expire(JSRuntime *rt, bool releaseAll);
848 :
849 : /* Must be called with the GC lock taken. */
850 : void expireAndFree(JSRuntime *rt, bool releaseAll);
851 :
852 : /* Must be called either during the GC or with the GC lock taken. */
853 : JS_FRIEND_API(int64_t) countCleanDecommittedArenas(JSRuntime *rt);
854 : };
855 :
856 : inline uintptr_t
857 -1 : Cell::address() const
858 : {
859 -1 : uintptr_t addr = uintptr_t(this);
860 -1 : JS_ASSERT(addr % Cell::CellSize == 0);
861 -1 : JS_ASSERT(Chunk::withinArenasRange(addr));
862 -1 : return addr;
863 : }
864 :
865 : inline ArenaHeader *
866 -1 : Cell::arenaHeader() const
867 : {
868 -1 : uintptr_t addr = address();
869 -1 : addr &= ~ArenaMask;
870 -1 : return reinterpret_cast<ArenaHeader *>(addr);
871 : }
872 :
873 : Chunk *
874 1009018784 : Cell::chunk() const
875 : {
876 1009018784 : uintptr_t addr = uintptr_t(this);
877 1009018784 : JS_ASSERT(addr % Cell::CellSize == 0);
878 1009018784 : addr &= ~(ChunkSize - 1);
879 1009018784 : return reinterpret_cast<Chunk *>(addr);
880 : }
881 :
882 : AllocKind
883 183095292 : Cell::getAllocKind() const
884 : {
885 183095292 : return arenaHeader()->getAllocKind();
886 : }
887 :
888 : #ifdef DEBUG
889 : inline bool
890 263204738 : Cell::isAligned() const
891 : {
892 263204738 : return Arena::isAligned(address(), arenaHeader()->getThingSize());
893 : }
894 : #endif
895 :
896 : inline uintptr_t
897 43385703 : ArenaHeader::address() const
898 : {
899 43385703 : uintptr_t addr = reinterpret_cast<uintptr_t>(this);
900 43385703 : JS_ASSERT(!(addr & ArenaMask));
901 43385703 : JS_ASSERT(Chunk::withinArenasRange(addr));
902 43385417 : return addr;
903 : }
904 :
905 : inline Chunk *
906 1685575 : ArenaHeader::chunk() const
907 : {
908 1685575 : return Chunk::fromAddress(address());
909 : }
910 :
911 : inline bool
912 85138 : ArenaHeader::isEmpty() const
913 : {
914 : /* Arena is empty if its first span covers the whole arena. */
915 85138 : JS_ASSERT(allocated());
916 85138 : size_t firstThingOffset = Arena::firstThingOffset(getAllocKind());
917 85138 : return firstFreeSpanOffsets == FreeSpan::encodeOffsets(firstThingOffset, ArenaMask);
918 : }
919 :
920 : inline size_t
921 286036969 : ArenaHeader::getThingSize() const
922 : {
923 286036969 : JS_ASSERT(allocated());
924 286036905 : return Arena::thingSize(getAllocKind());
925 : }
926 :
927 : inline ArenaHeader *
928 50996 : ArenaHeader::getNextDelayedMarking() const
929 : {
930 50996 : return &reinterpret_cast<Arena *>(nextDelayedMarking << ArenaShift)->aheader;
931 : }
932 :
933 : inline void
934 50996 : ArenaHeader::setNextDelayedMarking(ArenaHeader *aheader)
935 : {
936 50996 : JS_ASSERT(!(uintptr_t(aheader) & ArenaMask));
937 50996 : hasDelayedMarking = 1;
938 50996 : nextDelayedMarking = aheader->arenaAddress() >> ArenaShift;
939 50996 : }
940 :
941 : JS_ALWAYS_INLINE void
942 1009148140 : ChunkBitmap::getMarkWordAndMask(const Cell *cell, uint32_t color,
943 : uintptr_t **wordp, uintptr_t *maskp)
944 : {
945 1009148140 : size_t bit = (cell->address() & ChunkMask) / Cell::CellSize + color;
946 1009148140 : JS_ASSERT(bit < ArenaBitmapBits * ArenasPerChunk);
947 1009148140 : *maskp = uintptr_t(1) << (bit % JS_BITS_PER_WORD);
948 1009148140 : *wordp = &bitmap[bit / JS_BITS_PER_WORD];
949 1009148140 : }
950 :
951 : static void
952 1009018784 : AssertValidColor(const void *thing, uint32_t color)
953 : {
954 : #ifdef DEBUG
955 1009018784 : ArenaHeader *aheader = reinterpret_cast<const js::gc::Cell *>(thing)->arenaHeader();
956 1009018784 : JS_ASSERT_IF(color, color < aheader->getThingSize() / Cell::CellSize);
957 : #endif
958 1009018784 : }
959 :
960 : inline bool
961 642301283 : Cell::isMarked(uint32_t color) const
962 : {
963 642301283 : AssertValidColor(this, color);
964 642301283 : return chunk()->bitmap.isMarked(this, color);
965 : }
966 :
967 : bool
968 366717501 : Cell::markIfUnmarked(uint32_t color) const
969 : {
970 366717501 : AssertValidColor(this, color);
971 366717501 : return chunk()->bitmap.markIfUnmarked(this, color);
972 : }
973 :
974 : void
975 : Cell::unmark(uint32_t color) const
976 : {
977 : JS_ASSERT(color != BLACK);
978 : AssertValidColor(this, color);
979 : chunk()->bitmap.unmark(this, color);
980 : }
981 :
982 : JSCompartment *
983 -1033425325 : Cell::compartment() const
984 : {
985 -1033425325 : return arenaHeader()->compartment;
986 : }
987 :
988 : static inline JSGCTraceKind
989 79852964 : MapAllocToTraceKind(AllocKind thingKind)
990 : {
991 : static const JSGCTraceKind map[FINALIZE_LIMIT] = {
992 : JSTRACE_OBJECT, /* FINALIZE_OBJECT0 */
993 : JSTRACE_OBJECT, /* FINALIZE_OBJECT0_BACKGROUND */
994 : JSTRACE_OBJECT, /* FINALIZE_OBJECT2 */
995 : JSTRACE_OBJECT, /* FINALIZE_OBJECT2_BACKGROUND */
996 : JSTRACE_OBJECT, /* FINALIZE_OBJECT4 */
997 : JSTRACE_OBJECT, /* FINALIZE_OBJECT4_BACKGROUND */
998 : JSTRACE_OBJECT, /* FINALIZE_OBJECT8 */
999 : JSTRACE_OBJECT, /* FINALIZE_OBJECT8_BACKGROUND */
1000 : JSTRACE_OBJECT, /* FINALIZE_OBJECT12 */
1001 : JSTRACE_OBJECT, /* FINALIZE_OBJECT12_BACKGROUND */
1002 : JSTRACE_OBJECT, /* FINALIZE_OBJECT16 */
1003 : JSTRACE_OBJECT, /* FINALIZE_OBJECT16_BACKGROUND */
1004 : JSTRACE_SCRIPT, /* FINALIZE_SCRIPT */
1005 : JSTRACE_SHAPE, /* FINALIZE_SHAPE */
1006 : JSTRACE_BASE_SHAPE, /* FINALIZE_BASE_SHAPE */
1007 : JSTRACE_TYPE_OBJECT,/* FINALIZE_TYPE_OBJECT */
1008 : #if JS_HAS_XML_SUPPORT /* FINALIZE_XML */
1009 : JSTRACE_XML,
1010 : #endif
1011 : JSTRACE_STRING, /* FINALIZE_SHORT_STRING */
1012 : JSTRACE_STRING, /* FINALIZE_STRING */
1013 : JSTRACE_STRING, /* FINALIZE_EXTERNAL_STRING */
1014 : };
1015 79852964 : return map[thingKind];
1016 : }
1017 :
1018 : inline JSGCTraceKind
1019 : GetGCThingTraceKind(const void *thing);
1020 :
1021 : struct ArenaLists {
1022 :
1023 : /*
1024 : * ArenaList::head points to the start of the list. Normally cursor points
1025 : * to the first arena in the list with some free things and all arenas
1026 : * before cursor are fully allocated. However, as the arena currently being
1027 : * allocated from is considered full while its list of free spans is moved
1028 : * into the freeList, during the GC or cell enumeration, when an
1029 : * unallocated freeList is moved back to the arena, we can see an arena
1030 : * with some free cells before the cursor. The cursor is an indirect
1031 : * pointer to allow for efficient list insertion at the cursor point and
1032 : * other list manipulations.
1033 : */
1034 : struct ArenaList {
1035 : ArenaHeader *head;
1036 : ArenaHeader **cursor;
1037 :
1038 2300972 : ArenaList() {
1039 2300972 : clear();
1040 2300972 : }
1041 :
1042 2457920 : void clear() {
1043 2457920 : head = NULL;
1044 2457920 : cursor = &head;
1045 2457920 : }
1046 : };
1047 :
1048 : private:
1049 : /*
1050 : * For each arena kind its free list is represented as the first span with
1051 : * free things. Initially all the spans are initialized as empty. After we
1052 : * find a new arena with available things we move its first free span into
1053 : * the list and set the arena as fully allocated. way we do not need to
1054 : * update the arena header after the initial allocation. When starting the
1055 : * GC we only move the head of the of the list of spans back to the arena
1056 : * only for the arena that was not fully allocated.
1057 : */
1058 : FreeSpan freeLists[FINALIZE_LIMIT];
1059 :
1060 : ArenaList arenaLists[FINALIZE_LIMIT];
1061 :
1062 : #ifdef JS_THREADSAFE
1063 : /*
1064 : * The background finalization adds the finalized arenas to the list at
1065 : * the *cursor position. backgroundFinalizeState controls the interaction
1066 : * between the GC lock and the access to the list from the allocation
1067 : * thread.
1068 : *
1069 : * BFS_DONE indicates that the finalizations is not running or cannot
1070 : * affect this arena list. The allocation thread can access the list
1071 : * outside the GC lock.
1072 : *
1073 : * In BFS_RUN and BFS_JUST_FINISHED the allocation thread must take the
1074 : * lock. The former indicates that the finalization still runs. The latter
1075 : * signals that finalization just added to the list finalized arenas. In
1076 : * that case the lock effectively serves as a read barrier to ensure that
1077 : * the allocation thread see all the writes done during finalization.
1078 : */
1079 : enum BackgroundFinalizeState {
1080 : BFS_DONE,
1081 : BFS_RUN,
1082 : BFS_JUST_FINISHED
1083 : };
1084 :
1085 : volatile uintptr_t backgroundFinalizeState[FINALIZE_LIMIT];
1086 : #endif
1087 :
1088 : public:
1089 41285 : ArenaLists() {
1090 866985 : for (size_t i = 0; i != FINALIZE_LIMIT; ++i)
1091 825700 : freeLists[i].initAsEmpty();
1092 : #ifdef JS_THREADSAFE
1093 866985 : for (size_t i = 0; i != FINALIZE_LIMIT; ++i)
1094 825700 : backgroundFinalizeState[i] = BFS_DONE;
1095 : #endif
1096 41285 : }
1097 :
1098 41285 : ~ArenaLists() {
1099 1733970 : for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
1100 : #ifdef JS_THREADSAFE
1101 : /*
1102 : * We can only call this during the shutdown after the last GC when
1103 : * the background finalization is disabled.
1104 : */
1105 825700 : JS_ASSERT(backgroundFinalizeState[i] == BFS_DONE);
1106 : #endif
1107 825700 : ArenaHeader **headp = &arenaLists[i].head;
1108 1538658 : while (ArenaHeader *aheader = *headp) {
1109 356479 : *headp = aheader->next;
1110 356479 : aheader->chunk()->releaseArena(aheader);
1111 : }
1112 : }
1113 41285 : }
1114 :
1115 9942003 : const FreeSpan *getFreeList(AllocKind thingKind) const {
1116 9942003 : return &freeLists[thingKind];
1117 : }
1118 :
1119 352014 : ArenaHeader *getFirstArena(AllocKind thingKind) const {
1120 352014 : return arenaLists[thingKind].head;
1121 : }
1122 :
1123 45414 : bool arenaListsAreEmpty() const {
1124 518685 : for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
1125 : #ifdef JS_THREADSAFE
1126 : /*
1127 : * The arena cannot be empty if the background finalization is not yet
1128 : * done.
1129 : */
1130 496161 : if (backgroundFinalizeState[i] != BFS_DONE)
1131 22773 : return false;
1132 : #endif
1133 473388 : if (arenaLists[i].head)
1134 117 : return false;
1135 : }
1136 22524 : return true;
1137 : }
1138 :
1139 : #ifdef DEBUG
1140 117 : bool checkArenaListAllUnmarked() const {
1141 2457 : for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
1142 : # ifdef JS_THREADSAFE
1143 : /* The background finalization must have stopped at this point. */
1144 2550 : JS_ASSERT(backgroundFinalizeState[i] == BFS_DONE ||
1145 2550 : backgroundFinalizeState[i] == BFS_JUST_FINISHED);
1146 : # endif
1147 4588 : for (ArenaHeader *aheader = arenaLists[i].head; aheader; aheader = aheader->next) {
1148 2248 : if (!aheader->chunk()->bitmap.noBitsSet(aheader))
1149 0 : return false;
1150 : }
1151 : }
1152 117 : return true;
1153 : }
1154 : #endif
1155 :
1156 : #ifdef JS_THREADSAFE
1157 : bool doneBackgroundFinalize(AllocKind kind) const {
1158 : return backgroundFinalizeState[kind] == BFS_DONE;
1159 : }
1160 : #endif
1161 :
1162 : /*
1163 : * Return the free list back to the arena so the GC finalization will not
1164 : * run the finalizers over unitialized bytes from free things.
1165 : */
1166 83697 : void purge() {
1167 1757637 : for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
1168 1673940 : FreeSpan *headSpan = &freeLists[i];
1169 1673940 : if (!headSpan->isEmpty()) {
1170 408799 : ArenaHeader *aheader = headSpan->arenaHeader();
1171 408799 : aheader->setFirstFreeSpan(headSpan);
1172 408799 : headSpan->initAsEmpty();
1173 : }
1174 : }
1175 83697 : }
1176 :
1177 : inline void prepareForIncrementalGC(JSRuntime *rt);
1178 :
1179 : /*
1180 : * Temporarily copy the free list heads to the arenas so the code can see
1181 : * the proper value in ArenaHeader::freeList when accessing the latter
1182 : * outside the GC.
1183 : */
1184 95817 : void copyFreeListsToArenas() {
1185 2012157 : for (size_t i = 0; i != FINALIZE_LIMIT; ++i)
1186 1916340 : copyFreeListToArena(AllocKind(i));
1187 95817 : }
1188 :
1189 1947053 : void copyFreeListToArena(AllocKind thingKind) {
1190 1947053 : FreeSpan *headSpan = &freeLists[thingKind];
1191 1947053 : if (!headSpan->isEmpty()) {
1192 552172 : ArenaHeader *aheader = headSpan->arenaHeader();
1193 552172 : JS_ASSERT(!aheader->hasFreeThings());
1194 552172 : aheader->setFirstFreeSpan(headSpan);
1195 : }
1196 1947053 : }
1197 :
1198 : /*
1199 : * Clear the free lists in arenas that were temporarily set there using
1200 : * copyToArenas.
1201 : */
1202 73293 : void clearFreeListsInArenas() {
1203 1539153 : for (size_t i = 0; i != FINALIZE_LIMIT; ++i)
1204 1465860 : clearFreeListInArena(AllocKind(i));
1205 73293 : }
1206 :
1207 :
1208 1496573 : void clearFreeListInArena(AllocKind kind) {
1209 1496573 : FreeSpan *headSpan = &freeLists[kind];
1210 1496573 : if (!headSpan->isEmpty()) {
1211 143373 : ArenaHeader *aheader = headSpan->arenaHeader();
1212 143373 : JS_ASSERT(aheader->getFirstFreeSpan().isSameNonEmptySpan(headSpan));
1213 143373 : aheader->setAsFullyUsed();
1214 : }
1215 1496573 : }
1216 :
1217 : /*
1218 : * Check that the free list is either empty or were synchronized with the
1219 : * arena using copyToArena().
1220 : */
1221 321665 : bool isSynchronizedFreeList(AllocKind kind) {
1222 321665 : FreeSpan *headSpan = &freeLists[kind];
1223 321665 : if (headSpan->isEmpty())
1224 252500 : return true;
1225 69165 : ArenaHeader *aheader = headSpan->arenaHeader();
1226 69165 : if (aheader->hasFreeThings()) {
1227 : /*
1228 : * If the arena has a free list, it must be the same as one in
1229 : * lists.
1230 : */
1231 38452 : JS_ASSERT(aheader->getFirstFreeSpan().isSameNonEmptySpan(headSpan));
1232 38452 : return true;
1233 : }
1234 30713 : return false;
1235 : }
1236 :
1237 214032028 : JS_ALWAYS_INLINE void *allocateFromFreeList(AllocKind thingKind, size_t thingSize) {
1238 214032028 : return freeLists[thingKind].allocate(thingSize);
1239 : }
1240 :
1241 : static void *refillFreeList(JSContext *cx, AllocKind thingKind);
1242 :
1243 22524 : void checkEmptyFreeLists() {
1244 : #ifdef DEBUG
1245 473004 : for (size_t i = 0; i < mozilla::ArrayLength(freeLists); ++i)
1246 450480 : JS_ASSERT(freeLists[i].isEmpty());
1247 : #endif
1248 22524 : }
1249 :
1250 : void checkEmptyFreeList(AllocKind kind) {
1251 : JS_ASSERT(freeLists[kind].isEmpty());
1252 : }
1253 :
1254 : void finalizeObjects(FreeOp *fop);
1255 : void finalizeStrings(FreeOp *fop);
1256 : void finalizeShapes(FreeOp *fop);
1257 : void finalizeScripts(FreeOp *fop);
1258 :
1259 : #ifdef JS_THREADSAFE
1260 : static void backgroundFinalize(FreeOp *fop, ArenaHeader *listHead);
1261 : #endif
1262 :
1263 : private:
1264 : inline void finalizeNow(FreeOp *fop, AllocKind thingKind);
1265 : inline void finalizeLater(FreeOp *fop, AllocKind thingKind);
1266 :
1267 : inline void *allocateFromArena(JSCompartment *comp, AllocKind thingKind);
1268 : };
1269 :
1270 : /*
1271 : * Initial allocation size for data structures holding chunks is set to hold
1272 : * chunks with total capacity of 16MB to avoid buffer resizes during browser
1273 : * startup.
1274 : */
1275 : const size_t INITIAL_CHUNK_CAPACITY = 16 * 1024 * 1024 / ChunkSize;
1276 :
1277 : /* The number of GC cycles an empty chunk can survive before been released. */
1278 : const size_t MAX_EMPTY_CHUNK_AGE = 4;
1279 :
1280 : inline Cell *
1281 : AsCell(JSObject *obj)
1282 : {
1283 : return reinterpret_cast<Cell *>(obj);
1284 : }
1285 :
1286 : } /* namespace gc */
1287 :
1288 : struct GCPtrHasher
1289 : {
1290 : typedef void *Lookup;
1291 :
1292 0 : static HashNumber hash(void *key) {
1293 0 : return HashNumber(uintptr_t(key) >> JS_GCTHING_ZEROBITS);
1294 : }
1295 :
1296 0 : static bool match(void *l, void *k) { return l == k; }
1297 : };
1298 :
1299 : typedef HashMap<void *, uint32_t, GCPtrHasher, SystemAllocPolicy> GCLocks;
1300 :
1301 : struct RootInfo {
1302 19230101 : RootInfo() {}
1303 18837 : RootInfo(const char *name, JSGCRootType type) : name(name), type(type) {}
1304 : const char *name;
1305 : JSGCRootType type;
1306 : };
1307 :
1308 : typedef js::HashMap<void *,
1309 : RootInfo,
1310 : js::DefaultHasher<void *>,
1311 : js::SystemAllocPolicy> RootedValueMap;
1312 :
1313 : } /* namespace js */
1314 :
1315 : extern JS_FRIEND_API(JSGCTraceKind)
1316 : js_GetGCThingTraceKind(void *thing);
1317 :
1318 : extern JSBool
1319 : js_InitGC(JSRuntime *rt, uint32_t maxbytes);
1320 :
1321 : extern void
1322 : js_FinishGC(JSRuntime *rt);
1323 :
1324 : extern JSBool
1325 : js_AddRoot(JSContext *cx, js::Value *vp, const char *name);
1326 :
1327 : extern JSBool
1328 : js_AddGCThingRoot(JSContext *cx, void **rp, const char *name);
1329 :
1330 : #ifdef DEBUG
1331 : extern void
1332 : js_DumpNamedRoots(JSRuntime *rt,
1333 : void (*dump)(const char *name, void *rp, JSGCRootType type, void *data),
1334 : void *data);
1335 : #endif
1336 :
1337 : extern uint32_t
1338 : js_MapGCRoots(JSRuntime *rt, JSGCRootMapFun map, void *data);
1339 :
1340 : /* Table of pointers with count valid members. */
1341 : typedef struct JSPtrTable {
1342 : size_t count;
1343 : void **array;
1344 : } JSPtrTable;
1345 :
1346 : extern JSBool
1347 : js_LockGCThingRT(JSRuntime *rt, void *thing);
1348 :
1349 : extern void
1350 : js_UnlockGCThingRT(JSRuntime *rt, void *thing);
1351 :
1352 : extern JS_FRIEND_API(bool)
1353 : IsAboutToBeFinalized(const js::gc::Cell *thing);
1354 :
1355 : extern bool
1356 : IsAboutToBeFinalized(const js::Value &value);
1357 :
1358 : extern bool
1359 : js_IsAddressableGCThing(JSRuntime *rt, uintptr_t w, js::gc::AllocKind *thingKind, void **thing);
1360 :
1361 : namespace js {
1362 :
1363 : extern void
1364 : MarkCompartmentActive(js::StackFrame *fp);
1365 :
1366 : extern void
1367 : TraceRuntime(JSTracer *trc);
1368 :
1369 : extern JS_FRIEND_API(void)
1370 : MarkContext(JSTracer *trc, JSContext *acx);
1371 :
1372 : /* Must be called with GC lock taken. */
1373 : extern void
1374 : TriggerGC(JSRuntime *rt, js::gcreason::Reason reason);
1375 :
1376 : /* Must be called with GC lock taken. */
1377 : extern void
1378 : TriggerCompartmentGC(JSCompartment *comp, js::gcreason::Reason reason);
1379 :
1380 : extern void
1381 : MaybeGC(JSContext *cx);
1382 :
1383 : extern void
1384 : ShrinkGCBuffers(JSRuntime *rt);
1385 :
1386 : extern void
1387 : PrepareForFullGC(JSRuntime *rt);
1388 :
1389 : /*
1390 : * Kinds of js_GC invocation.
1391 : */
1392 : typedef enum JSGCInvocationKind {
1393 : /* Normal invocation. */
1394 : GC_NORMAL = 0,
1395 :
1396 : /* Minimize GC triggers and release empty GC chunks right away. */
1397 : GC_SHRINK = 1
1398 : } JSGCInvocationKind;
1399 :
1400 : extern void
1401 : GC(JSContext *cx, JSGCInvocationKind gckind, js::gcreason::Reason reason);
1402 :
1403 : extern void
1404 : GCSlice(JSContext *cx, JSGCInvocationKind gckind, js::gcreason::Reason reason);
1405 :
1406 : extern void
1407 : GCDebugSlice(JSContext *cx, bool limit, int64_t objCount);
1408 :
1409 : extern void
1410 : PrepareForDebugGC(JSRuntime *rt);
1411 :
1412 : } /* namespace js */
1413 :
1414 : namespace js {
1415 :
1416 : void
1417 : InitTracer(JSTracer *trc, JSRuntime *rt, JSTraceCallback callback);
1418 :
1419 : #ifdef JS_THREADSAFE
1420 :
1421 18761 : class GCHelperThread {
1422 : enum State {
1423 : IDLE,
1424 : SWEEPING,
1425 : ALLOCATING,
1426 : CANCEL_ALLOCATION,
1427 : SHUTDOWN
1428 : };
1429 :
1430 : /*
1431 : * During the finalization we do not free immediately. Rather we add the
1432 : * corresponding pointers to a buffer which we later release on a
1433 : * separated thread.
1434 : *
1435 : * The buffer is implemented as a vector of 64K arrays of pointers, not as
1436 : * a simple vector, to avoid realloc calls during the vector growth and to
1437 : * not bloat the binary size of the inlined freeLater method. Any OOM
1438 : * during buffer growth results in the pointer being freed immediately.
1439 : */
1440 : static const size_t FREE_ARRAY_SIZE = size_t(1) << 16;
1441 : static const size_t FREE_ARRAY_LENGTH = FREE_ARRAY_SIZE / sizeof(void *);
1442 :
1443 : JSRuntime *const rt;
1444 : PRThread *thread;
1445 : PRCondVar *wakeup;
1446 : PRCondVar *done;
1447 : volatile State state;
1448 :
1449 : JSContext *finalizationContext;
1450 : bool shrinkFlag;
1451 :
1452 : Vector<void **, 16, js::SystemAllocPolicy> freeVector;
1453 : void **freeCursor;
1454 : void **freeCursorEnd;
1455 :
1456 : Vector<js::gc::ArenaHeader *, 64, js::SystemAllocPolicy> finalizeVector;
1457 :
1458 : bool backgroundAllocation;
1459 :
1460 : friend struct js::gc::ArenaLists;
1461 :
1462 : JS_FRIEND_API(void)
1463 : replenishAndFreeLater(void *ptr);
1464 :
1465 12426 : static void freeElementsAndArray(void **array, void **end) {
1466 12426 : JS_ASSERT(array <= end);
1467 101742 : for (void **p = array; p != end; ++p)
1468 89316 : js::Foreground::free_(*p);
1469 12426 : js::Foreground::free_(array);
1470 12426 : }
1471 :
1472 : static void threadMain(void* arg);
1473 : void threadLoop();
1474 :
1475 : /* Must be called with the GC lock taken. */
1476 : void doSweep();
1477 :
1478 : public:
1479 18761 : GCHelperThread(JSRuntime *rt)
1480 : : rt(rt),
1481 : thread(NULL),
1482 : wakeup(NULL),
1483 : done(NULL),
1484 : state(IDLE),
1485 : finalizationContext(NULL),
1486 : shrinkFlag(false),
1487 : freeCursor(NULL),
1488 : freeCursorEnd(NULL),
1489 18761 : backgroundAllocation(true)
1490 18761 : { }
1491 :
1492 : bool init();
1493 : void finish();
1494 :
1495 : /* Must be called with the GC lock taken. */
1496 : void startBackgroundSweep(JSContext *cx, bool shouldShrink);
1497 :
1498 : /* Must be called with the GC lock taken. */
1499 : void startBackgroundShrink();
1500 :
1501 : /* Must be called with the GC lock taken. */
1502 : void waitBackgroundSweepEnd();
1503 :
1504 : /* Must be called with the GC lock taken. */
1505 : void waitBackgroundSweepOrAllocEnd();
1506 :
1507 : /* Must be called with the GC lock taken. */
1508 : inline void startBackgroundAllocationIfIdle();
1509 :
1510 42904 : bool canBackgroundAllocate() const {
1511 42904 : return backgroundAllocation;
1512 : }
1513 :
1514 : void disableBackgroundAllocation() {
1515 : backgroundAllocation = false;
1516 : }
1517 :
1518 : PRThread *getThread() const {
1519 : return thread;
1520 : }
1521 :
1522 : /*
1523 : * Outside the GC lock may give true answer when in fact the sweeping has
1524 : * been done.
1525 : */
1526 4125546 : bool sweeping() const {
1527 4125546 : return state == SWEEPING;
1528 : }
1529 :
1530 : bool shouldShrink() const {
1531 : JS_ASSERT(sweeping());
1532 : return shrinkFlag;
1533 : }
1534 :
1535 89316 : void freeLater(void *ptr) {
1536 89316 : JS_ASSERT(!sweeping());
1537 89316 : if (freeCursor != freeCursorEnd)
1538 76890 : *freeCursor++ = ptr;
1539 : else
1540 12426 : replenishAndFreeLater(ptr);
1541 89316 : }
1542 :
1543 : /* Must be called with the GC lock taken. */
1544 : bool prepareForBackgroundSweep();
1545 : };
1546 :
1547 : #endif /* JS_THREADSAFE */
1548 :
1549 : struct GCChunkHasher {
1550 : typedef gc::Chunk *Lookup;
1551 :
1552 : /*
1553 : * Strip zeros for better distribution after multiplying by the golden
1554 : * ratio.
1555 : */
1556 206734384 : static HashNumber hash(gc::Chunk *chunk) {
1557 206734384 : JS_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask));
1558 206734384 : return HashNumber(uintptr_t(chunk) >> gc::ChunkShift);
1559 : }
1560 :
1561 11599634 : static bool match(gc::Chunk *k, gc::Chunk *l) {
1562 11599634 : JS_ASSERT(!(uintptr_t(k) & gc::ChunkMask));
1563 11599634 : JS_ASSERT(!(uintptr_t(l) & gc::ChunkMask));
1564 11599634 : return k == l;
1565 : }
1566 : };
1567 :
1568 : typedef HashSet<js::gc::Chunk *, GCChunkHasher, SystemAllocPolicy> GCChunkSet;
1569 :
1570 : template<class T>
1571 : struct MarkStack {
1572 : T *stack;
1573 : T *tos;
1574 : T *limit;
1575 :
1576 : T *ballast;
1577 : T *ballastLimit;
1578 :
1579 : size_t sizeLimit;
1580 :
1581 18761 : MarkStack(size_t sizeLimit)
1582 : : stack(NULL),
1583 : tos(NULL),
1584 : limit(NULL),
1585 : ballast(NULL),
1586 : ballastLimit(NULL),
1587 18761 : sizeLimit(sizeLimit) { }
1588 :
1589 18761 : ~MarkStack() {
1590 18761 : if (stack != ballast)
1591 0 : js_free(stack);
1592 18761 : js_free(ballast);
1593 18761 : }
1594 :
1595 18761 : bool init(size_t ballastcap) {
1596 18761 : JS_ASSERT(!stack);
1597 :
1598 18761 : if (ballastcap == 0)
1599 0 : return true;
1600 :
1601 18761 : ballast = (T *)js_malloc(sizeof(T) * ballastcap);
1602 18761 : if (!ballast)
1603 0 : return false;
1604 18761 : ballastLimit = ballast + ballastcap;
1605 18761 : initFromBallast();
1606 18761 : return true;
1607 : }
1608 :
1609 60071 : void initFromBallast() {
1610 60071 : stack = ballast;
1611 60071 : limit = ballastLimit;
1612 60071 : if (size_t(limit - stack) > sizeLimit)
1613 0 : limit = stack + sizeLimit;
1614 60071 : tos = stack;
1615 60071 : }
1616 :
1617 0 : void setSizeLimit(size_t size) {
1618 0 : JS_ASSERT(isEmpty());
1619 :
1620 0 : sizeLimit = size;
1621 0 : reset();
1622 0 : }
1623 :
1624 4120959 : bool push(T item) {
1625 4120959 : if (tos == limit) {
1626 18 : if (!enlarge())
1627 0 : return false;
1628 : }
1629 4120959 : JS_ASSERT(tos < limit);
1630 4120959 : *tos++ = item;
1631 4120959 : return true;
1632 : }
1633 :
1634 11472047 : bool push(T item1, T item2, T item3) {
1635 11472047 : T *nextTos = tos + 3;
1636 11472047 : if (nextTos > limit) {
1637 0 : if (!enlarge())
1638 0 : return false;
1639 0 : nextTos = tos + 3;
1640 : }
1641 11472047 : JS_ASSERT(nextTos <= limit);
1642 11472047 : tos[0] = item1;
1643 11472047 : tos[1] = item2;
1644 11472047 : tos[2] = item3;
1645 11472047 : tos = nextTos;
1646 11472047 : return true;
1647 : }
1648 :
1649 54550636 : bool isEmpty() const {
1650 54550636 : return tos == stack;
1651 : }
1652 :
1653 38533602 : T pop() {
1654 38533602 : JS_ASSERT(!isEmpty());
1655 38533602 : return *--tos;
1656 : }
1657 :
1658 5784525 : ptrdiff_t position() const {
1659 5784525 : return tos - stack;
1660 : }
1661 :
1662 41310 : void reset() {
1663 41310 : if (stack != ballast)
1664 18 : js_free(stack);
1665 41310 : initFromBallast();
1666 41310 : JS_ASSERT(stack == ballast);
1667 41310 : }
1668 :
1669 18 : bool enlarge() {
1670 18 : size_t tosIndex = tos - stack;
1671 18 : size_t cap = limit - stack;
1672 18 : if (cap == sizeLimit)
1673 0 : return false;
1674 18 : size_t newcap = cap * 2;
1675 18 : if (newcap == 0)
1676 0 : newcap = 32;
1677 18 : if (newcap > sizeLimit)
1678 0 : newcap = sizeLimit;
1679 :
1680 : T *newStack;
1681 18 : if (stack == ballast) {
1682 18 : newStack = (T *)js_malloc(sizeof(T) * newcap);
1683 18 : if (!newStack)
1684 0 : return false;
1685 589860 : for (T *src = stack, *dst = newStack; src < tos; )
1686 589824 : *dst++ = *src++;
1687 : } else {
1688 0 : newStack = (T *)js_realloc(stack, sizeof(T) * newcap);
1689 0 : if (!newStack)
1690 0 : return false;
1691 : }
1692 18 : stack = newStack;
1693 18 : tos = stack + tosIndex;
1694 18 : limit = newStack + newcap;
1695 18 : return true;
1696 : }
1697 :
1698 0 : size_t sizeOfExcludingThis(JSMallocSizeOfFun mallocSizeOf) const {
1699 0 : size_t n = 0;
1700 0 : if (stack != ballast)
1701 0 : n += mallocSizeOf(stack);
1702 0 : n += mallocSizeOf(ballast);
1703 0 : return n;
1704 : }
1705 : };
1706 :
1707 : /*
1708 : * This class records how much work has been done in a given GC slice, so that
1709 : * we can return before pausing for too long. Some slices are allowed to run for
1710 : * unlimited time, and others are bounded. To reduce the number of gettimeofday
1711 : * calls, we only check the time every 1000 operations.
1712 : */
1713 : struct SliceBudget {
1714 : int64_t deadline; /* in microseconds */
1715 : intptr_t counter;
1716 :
1717 : static const intptr_t CounterReset = 1000;
1718 :
1719 : static const int64_t Unlimited = 0;
1720 : static int64_t TimeBudget(int64_t millis);
1721 : static int64_t WorkBudget(int64_t work);
1722 :
1723 : /* Equivalent to SliceBudget(UnlimitedBudget). */
1724 : SliceBudget();
1725 :
1726 : /* Instantiate as SliceBudget(Time/WorkBudget(n)). */
1727 : SliceBudget(int64_t budget);
1728 :
1729 0 : void reset() {
1730 0 : deadline = INT64_MAX;
1731 0 : counter = INTPTR_MAX;
1732 0 : }
1733 :
1734 69485312 : void step(intptr_t amt = 1) {
1735 69485312 : counter -= amt;
1736 69485312 : }
1737 :
1738 : bool checkOverBudget();
1739 :
1740 84729706 : bool isOverBudget() {
1741 84729706 : if (counter >= 0)
1742 84729607 : return false;
1743 99 : return checkOverBudget();
1744 : }
1745 : };
1746 :
1747 : static const size_t MARK_STACK_LENGTH = 32768;
1748 :
1749 18761 : struct GCMarker : public JSTracer {
1750 : private:
1751 : /*
1752 : * We use a common mark stack to mark GC things of different types and use
1753 : * the explicit tags to distinguish them when it cannot be deduced from
1754 : * the context of push or pop operation.
1755 : */
1756 : enum StackTag {
1757 : ValueArrayTag,
1758 : ObjectTag,
1759 : TypeTag,
1760 : XmlTag,
1761 : SavedValueArrayTag,
1762 : LastTag = SavedValueArrayTag
1763 : };
1764 :
1765 : static const uintptr_t StackTagMask = 7;
1766 :
1767 : static void staticAsserts() {
1768 : JS_STATIC_ASSERT(StackTagMask >= uintptr_t(LastTag));
1769 : JS_STATIC_ASSERT(StackTagMask <= gc::Cell::CellMask);
1770 : }
1771 :
1772 : public:
1773 : explicit GCMarker();
1774 : bool init();
1775 :
1776 0 : void setSizeLimit(size_t size) { stack.setSizeLimit(size); }
1777 0 : size_t sizeLimit() const { return stack.sizeLimit; }
1778 :
1779 : void start(JSRuntime *rt);
1780 : void stop();
1781 : void reset();
1782 :
1783 3697803 : void pushObject(JSObject *obj) {
1784 3697803 : pushTaggedPtr(ObjectTag, obj);
1785 3697803 : }
1786 :
1787 422962 : void pushType(types::TypeObject *type) {
1788 422962 : pushTaggedPtr(TypeTag, type);
1789 422962 : }
1790 :
1791 194 : void pushXML(JSXML *xml) {
1792 194 : pushTaggedPtr(XmlTag, xml);
1793 194 : }
1794 :
1795 112185007 : uint32_t getMarkColor() const {
1796 112185007 : return color;
1797 : }
1798 :
1799 : /*
1800 : * The only valid color transition during a GC is from black to gray. It is
1801 : * wrong to switch the mark color from gray to black. The reason is that the
1802 : * cycle collector depends on the invariant that there are no black to gray
1803 : * edges in the GC heap. This invariant lets the CC not trace through black
1804 : * objects. If this invariant is violated, the cycle collector may free
1805 : * objects that are still reachable.
1806 : */
1807 38436 : void setMarkColorGray() {
1808 38436 : JS_ASSERT(isDrained());
1809 38436 : JS_ASSERT(color == gc::BLACK);
1810 38436 : color = gc::GRAY;
1811 38436 : }
1812 :
1813 : inline void delayMarkingArena(gc::ArenaHeader *aheader);
1814 : void delayMarkingChildren(const void *thing);
1815 : void markDelayedChildren(gc::ArenaHeader *aheader);
1816 : bool markDelayedChildren(SliceBudget &budget);
1817 77201 : bool hasDelayedChildren() const {
1818 77201 : return !!unmarkedArenaStackTop;
1819 : }
1820 :
1821 348807 : bool isDrained() {
1822 348807 : return isMarkStackEmpty() && !unmarkedArenaStackTop;
1823 : }
1824 :
1825 : bool drainMarkStack(SliceBudget &budget);
1826 :
1827 : /*
1828 : * Gray marking must be done after all black marking is complete. However,
1829 : * we do not have write barriers on XPConnect roots. Therefore, XPConnect
1830 : * roots must be accumulated in the first slice of incremental GC. We
1831 : * accumulate these roots in the GrayRootMarker and then mark them later,
1832 : * after black marking is complete. This accumulation can fail, but in that
1833 : * case we switch to non-incremental GC.
1834 : */
1835 : bool hasBufferedGrayRoots() const;
1836 : void startBufferingGrayRoots();
1837 : void endBufferingGrayRoots();
1838 : void markBufferedGrayRoots();
1839 :
1840 : static void GrayCallback(JSTracer *trc, void **thing, JSGCTraceKind kind);
1841 :
1842 : size_t sizeOfExcludingThis(JSMallocSizeOfFun mallocSizeOf) const;
1843 :
1844 : MarkStack<uintptr_t> stack;
1845 :
1846 : private:
1847 : #ifdef DEBUG
1848 : void checkCompartment(void *p);
1849 : #else
1850 : void checkCompartment(void *p) {}
1851 : #endif
1852 :
1853 4120959 : void pushTaggedPtr(StackTag tag, void *ptr) {
1854 4120959 : checkCompartment(ptr);
1855 4120959 : uintptr_t addr = reinterpret_cast<uintptr_t>(ptr);
1856 4120959 : JS_ASSERT(!(addr & StackTagMask));
1857 4120959 : if (!stack.push(addr | uintptr_t(tag)))
1858 0 : delayMarkingChildren(ptr);
1859 4120959 : }
1860 :
1861 18604267 : void pushValueArray(JSObject *obj, void *start, void *end) {
1862 18604267 : checkCompartment(obj);
1863 :
1864 18604267 : if (start == end)
1865 7132220 : return;
1866 :
1867 11472047 : JS_ASSERT(start <= end);
1868 11472047 : uintptr_t tagged = reinterpret_cast<uintptr_t>(obj) | GCMarker::ValueArrayTag;
1869 11472047 : uintptr_t startAddr = reinterpret_cast<uintptr_t>(start);
1870 11472047 : uintptr_t endAddr = reinterpret_cast<uintptr_t>(end);
1871 :
1872 : /*
1873 : * Push in the reverse order so obj will be on top. If we cannot push
1874 : * the array, we trigger delay marking for the whole object.
1875 : */
1876 11472047 : if (!stack.push(endAddr, startAddr, tagged))
1877 0 : delayMarkingChildren(obj);
1878 : }
1879 :
1880 350253 : bool isMarkStackEmpty() {
1881 350253 : return stack.isEmpty();
1882 : }
1883 :
1884 : bool restoreValueArray(JSObject *obj, void **vpp, void **endp);
1885 : void saveValueRanges();
1886 : inline void processMarkStackTop(SliceBudget &budget);
1887 :
1888 : void appendGrayRoot(void *thing, JSGCTraceKind kind);
1889 :
1890 : /* The color is only applied to objects, functions and xml. */
1891 : uint32_t color;
1892 :
1893 : DebugOnly<bool> started;
1894 :
1895 : /* Pointer to the top of the stack of arenas we are delaying marking on. */
1896 : js::gc::ArenaHeader *unmarkedArenaStackTop;
1897 : /* Count of arenas that are currently in the stack. */
1898 : DebugOnly<size_t> markLaterArenas;
1899 :
1900 0 : struct GrayRoot {
1901 : void *thing;
1902 : JSGCTraceKind kind;
1903 : #ifdef DEBUG
1904 : JSTraceNamePrinter debugPrinter;
1905 : const void *debugPrintArg;
1906 : size_t debugPrintIndex;
1907 : #endif
1908 :
1909 0 : GrayRoot(void *thing, JSGCTraceKind kind)
1910 0 : : thing(thing), kind(kind) {}
1911 : };
1912 :
1913 : bool grayFailed;
1914 : Vector<GrayRoot, 0, SystemAllocPolicy> grayRoots;
1915 : };
1916 :
1917 : void
1918 : SetMarkStackLimit(JSRuntime *rt, size_t limit);
1919 :
1920 : void
1921 : MarkStackRangeConservatively(JSTracer *trc, Value *begin, Value *end);
1922 :
1923 : typedef void (*IterateChunkCallback)(JSRuntime *rt, void *data, gc::Chunk *chunk);
1924 : typedef void (*IterateArenaCallback)(JSRuntime *rt, void *data, gc::Arena *arena,
1925 : JSGCTraceKind traceKind, size_t thingSize);
1926 : typedef void (*IterateCellCallback)(JSRuntime *rt, void *data, void *thing,
1927 : JSGCTraceKind traceKind, size_t thingSize);
1928 :
1929 : /*
1930 : * This function calls |compartmentCallback| on every compartment,
1931 : * |arenaCallback| on every in-use arena, and |cellCallback| on every in-use
1932 : * cell in the GC heap.
1933 : */
1934 : extern JS_FRIEND_API(void)
1935 : IterateCompartmentsArenasCells(JSRuntime *rt, void *data,
1936 : JSIterateCompartmentCallback compartmentCallback,
1937 : IterateArenaCallback arenaCallback,
1938 : IterateCellCallback cellCallback);
1939 :
1940 : /*
1941 : * Invoke chunkCallback on every in-use chunk.
1942 : */
1943 : extern JS_FRIEND_API(void)
1944 : IterateChunks(JSRuntime *rt, void *data, IterateChunkCallback chunkCallback);
1945 :
1946 : /*
1947 : * Invoke cellCallback on every in-use object of the specified thing kind for
1948 : * the given compartment or for all compartments if it is null.
1949 : */
1950 : extern JS_FRIEND_API(void)
1951 : IterateCells(JSRuntime *rt, JSCompartment *compartment, gc::AllocKind thingKind,
1952 : void *data, IterateCellCallback cellCallback);
1953 :
1954 : } /* namespace js */
1955 :
1956 : extern void
1957 : js_FinalizeStringRT(JSRuntime *rt, JSString *str);
1958 :
1959 : /*
1960 : * Macro to test if a traversal is the marking phase of the GC.
1961 : */
1962 : #define IS_GC_MARKING_TRACER(trc) \
1963 : ((trc)->callback == NULL || (trc)->callback == GCMarker::GrayCallback)
1964 :
1965 : namespace js {
1966 : namespace gc {
1967 :
1968 : JSCompartment *
1969 : NewCompartment(JSContext *cx, JSPrincipals *principals);
1970 :
1971 : /* Tries to run a GC no matter what (used for GC zeal). */
1972 : void
1973 : RunDebugGC(JSContext *cx);
1974 :
1975 : void
1976 : SetDeterministicGC(JSContext *cx, bool enabled);
1977 :
1978 : #if defined(JSGC_ROOT_ANALYSIS) && defined(DEBUG) && !defined(JS_THREADSAFE)
1979 : /* Overwrites stack references to GC things which have not been rooted. */
1980 : void CheckStackRoots(JSContext *cx);
1981 :
1982 : inline void MaybeCheckStackRoots(JSContext *cx) { CheckStackRoots(cx); }
1983 : #else
1984 199062626 : inline void MaybeCheckStackRoots(JSContext *cx) {}
1985 : #endif
1986 :
1987 : const int ZealPokeValue = 1;
1988 : const int ZealAllocValue = 2;
1989 : const int ZealFrameGCValue = 3;
1990 : const int ZealVerifierValue = 4;
1991 : const int ZealFrameVerifierValue = 5;
1992 :
1993 : #ifdef JS_GC_ZEAL
1994 :
1995 : /* Check that write barriers have been used correctly. See jsgc.cpp. */
1996 : void
1997 : VerifyBarriers(JSContext *cx);
1998 :
1999 : void
2000 : MaybeVerifyBarriers(JSContext *cx, bool always = false);
2001 :
2002 : #else
2003 :
2004 : static inline void
2005 : VerifyBarriers(JSContext *cx)
2006 : {
2007 : }
2008 :
2009 : static inline void
2010 : MaybeVerifyBarriers(JSContext *cx, bool always = false)
2011 : {
2012 : }
2013 :
2014 : #endif
2015 :
2016 : } /* namespace gc */
2017 :
2018 : static inline JSCompartment *
2019 : GetObjectCompartment(JSObject *obj) { return reinterpret_cast<js::gc::Cell *>(obj)->compartment(); }
2020 :
2021 : } /* namespace js */
2022 :
2023 : #endif /* jsgc_h___ */
|