1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : */
3 : /* This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "jsgcmark.h"
8 : #include "jsprf.h"
9 : #include "jsscope.h"
10 : #include "jsstr.h"
11 :
12 : #include "jsobjinlines.h"
13 : #include "jsscopeinlines.h"
14 :
15 : #include "vm/String-inl.h"
16 : #include "methodjit/MethodJIT.h"
17 :
18 : /*
19 : * There are two mostly separate mark paths. The first is a fast path used
20 : * internally in the GC. The second is a slow path used for root marking and
21 : * for API consumers like the cycle collector or Class::trace implementations.
22 : *
23 : * The fast path uses explicit stacks. The basic marking process during a GC is
24 : * that all roots are pushed on to a mark stack, and then each item on the
25 : * stack is scanned (possibly pushing more stuff) until the stack is empty.
26 : *
27 : * PushMarkStack pushes a GC thing onto the mark stack. In some cases (shapes
28 : * or strings) it eagerly marks the object rather than pushing it. Popping and
29 : * scanning is done by the processMarkStackTop method. For efficiency reasons
30 : * like tail recursion elimination that method also implements the scanning of
31 : * objects. For other GC things it uses helper methods.
32 : *
33 : * Most of the marking code outside jsgcmark uses functions like MarkObject,
34 : * MarkString, etc. These functions check if an object is in the compartment
35 : * currently being GCed. If it is, they call PushMarkStack. Roots are pushed
36 : * this way as well as pointers traversed inside trace hooks (for things like
37 : * IteratorClass). It it always valid to call a MarkX function instead of
38 : * PushMarkStack, although it may be slower.
39 : *
40 : * The MarkX functions also handle non-GC object traversal. In this case, they
41 : * call a callback for each object visited. This is a recursive process; the
42 : * mark stacks are not involved. These callbacks may ask for the outgoing
43 : * pointers to be visited. Eventually, this leads to the MarkChildren functions
44 : * being called. These functions duplicate much of the functionality of
45 : * scanning functions, but they don't push onto an explicit stack.
46 : */
47 :
48 : namespace js {
49 : namespace gc {
50 :
51 : static inline void
52 : PushMarkStack(GCMarker *gcmarker, JSXML *thing);
53 :
54 : static inline void
55 : PushMarkStack(GCMarker *gcmarker, JSObject *thing);
56 :
57 : static inline void
58 : PushMarkStack(GCMarker *gcmarker, JSFunction *thing);
59 :
60 : static inline void
61 : PushMarkStack(GCMarker *gcmarker, JSScript *thing);
62 :
63 : static inline void
64 : PushMarkStack(GCMarker *gcmarker, Shape *thing);
65 :
66 : static inline void
67 : PushMarkStack(GCMarker *gcmarker, JSString *thing);
68 :
69 : static inline void
70 : PushMarkStack(GCMarker *gcmarker, types::TypeObject *thing);
71 :
72 : /*** Object Marking ***/
73 :
74 : template<typename T>
75 : static inline void
76 458804796 : CheckMarkedThing(JSTracer *trc, T *thing)
77 : {
78 458804796 : JS_ASSERT(trc);
79 458804796 : JS_ASSERT(thing);
80 458804796 : JS_ASSERT(trc->debugPrinter || trc->debugPrintArg);
81 458804796 : JS_ASSERT_IF(trc->runtime->gcCurrentCompartment, IS_GC_MARKING_TRACER(trc));
82 :
83 458804796 : JS_ASSERT(thing->isAligned());
84 :
85 458804796 : JS_ASSERT(thing->compartment());
86 458804796 : JS_ASSERT(thing->compartment()->rt == trc->runtime);
87 458804796 : }
88 :
89 : template<typename T>
90 : void
91 458804796 : MarkInternal(JSTracer *trc, T *thing)
92 : {
93 458804796 : CheckMarkedThing(trc, thing);
94 :
95 458804796 : JSRuntime *rt = trc->runtime;
96 :
97 458804796 : JS_ASSERT_IF(rt->gcCheckCompartment,
98 : thing->compartment() == rt->gcCheckCompartment ||
99 : thing->compartment() == rt->atomsCompartment);
100 :
101 : /*
102 : * Don't mark things outside a compartment if we are in a per-compartment
103 : * GC.
104 : */
105 458804796 : if (!rt->gcCurrentCompartment || thing->compartment() == rt->gcCurrentCompartment) {
106 457716851 : if (!trc->callback) {
107 412238406 : PushMarkStack(static_cast<GCMarker *>(trc), thing);
108 : } else {
109 45478445 : void *tmp = (void *)thing;
110 45478445 : trc->callback(trc, &tmp, GetGCThingTraceKind(thing));
111 45478445 : JS_ASSERT(tmp == thing);
112 : }
113 : }
114 :
115 : #ifdef DEBUG
116 458804796 : trc->debugPrinter = NULL;
117 458804796 : trc->debugPrintArg = NULL;
118 : #endif
119 458804796 : }
120 :
121 : #define JS_ROOT_MARKING_ASSERT(trc) \
122 : JS_ASSERT_IF(IS_GC_MARKING_TRACER(trc), \
123 : trc->runtime->gcIncrementalState == NO_INCREMENTAL || \
124 : trc->runtime->gcIncrementalState == MARK_ROOTS);
125 :
126 :
127 : template <typename T>
128 : static void
129 370230871 : MarkUnbarriered(JSTracer *trc, T **thingp, const char *name)
130 : {
131 370230871 : JS_SET_TRACING_NAME(trc, name);
132 370230871 : MarkInternal(trc, *thingp);
133 370230871 : }
134 :
135 : template <typename T>
136 : static void
137 28818190 : Mark(JSTracer *trc, HeapPtr<T> *thing, const char *name)
138 : {
139 28818190 : JS_SET_TRACING_NAME(trc, name);
140 28818190 : MarkInternal(trc, thing->get());
141 28818190 : }
142 :
143 : template <typename T>
144 : static void
145 20720407 : MarkRoot(JSTracer *trc, T **thingp, const char *name)
146 : {
147 20720407 : JS_ROOT_MARKING_ASSERT(trc);
148 20720407 : JS_SET_TRACING_NAME(trc, name);
149 20720407 : MarkInternal(trc, *thingp);
150 20720407 : }
151 :
152 : template <typename T>
153 : static void
154 2277289 : MarkRange(JSTracer *trc, size_t len, HeapPtr<T> *vec, const char *name)
155 : {
156 6965922 : for (size_t i = 0; i < len; ++i) {
157 4688633 : if (T *obj = vec[i]) {
158 4688633 : JS_SET_TRACING_INDEX(trc, name, i);
159 4688633 : MarkInternal(trc, obj);
160 : }
161 : }
162 2277289 : }
163 :
164 : template <typename T>
165 : static void
166 596 : MarkRootRange(JSTracer *trc, size_t len, T **vec, const char *name)
167 : {
168 596 : JS_ROOT_MARKING_ASSERT(trc);
169 3404 : for (size_t i = 0; i < len; ++i) {
170 2808 : JS_SET_TRACING_INDEX(trc, name, i);
171 2808 : MarkInternal(trc, vec[i]);
172 : }
173 596 : }
174 :
175 : #define DeclMarkerImpl(base, type) \
176 : void \
177 : Mark##base(JSTracer *trc, HeapPtr<type> *thing, const char *name) \
178 : { \
179 : Mark<type>(trc, thing, name); \
180 : } \
181 : \
182 : void \
183 : Mark##base##Root(JSTracer *trc, type **thingp, const char *name) \
184 : { \
185 : MarkRoot<type>(trc, thingp, name); \
186 : } \
187 : \
188 : void \
189 : Mark##base##Unbarriered(JSTracer *trc, type **thingp, const char *name) \
190 : { \
191 : MarkUnbarriered<type>(trc, thingp, name); \
192 : } \
193 : \
194 : void Mark##base##Range(JSTracer *trc, size_t len, HeapPtr<type> *vec, const char *name) \
195 : { \
196 : MarkRange<type>(trc, len, vec, name); \
197 : } \
198 : \
199 : void Mark##base##RootRange(JSTracer *trc, size_t len, type **vec, const char *name) \
200 : { \
201 : MarkRootRange<type>(trc, len, vec, name); \
202 : } \
203 :
204 3706217 : DeclMarkerImpl(BaseShape, BaseShape)
205 38231 : DeclMarkerImpl(BaseShape, UnownedBaseShape)
206 1402 : DeclMarkerImpl(Object, ArgumentsObject)
207 66676 : DeclMarkerImpl(Object, GlobalObject)
208 12276747 : DeclMarkerImpl(Object, JSObject)
209 7439765 : DeclMarkerImpl(Object, JSFunction)
210 13739815 : DeclMarkerImpl(Script, JSScript)
211 13272763 : DeclMarkerImpl(Shape, Shape)
212 367944444 : DeclMarkerImpl(String, JSAtom)
213 160568 : DeclMarkerImpl(String, JSString)
214 7643 : DeclMarkerImpl(String, JSFlatString)
215 9602 : DeclMarkerImpl(String, JSLinearString)
216 3380910 : DeclMarkerImpl(TypeObject, types::TypeObject)
217 : #if JS_HAS_XML_SUPPORT
218 2570 : DeclMarkerImpl(XML, JSXML)
219 : #endif
220 :
221 : /*** Externally Typed Marking ***/
222 :
223 : void
224 30472618 : MarkKind(JSTracer *trc, void *thing, JSGCTraceKind kind)
225 : {
226 30472618 : JS_ASSERT(thing);
227 30472618 : JS_ASSERT(kind == GetGCThingTraceKind(thing));
228 30472618 : switch (kind) {
229 : case JSTRACE_OBJECT:
230 23842393 : MarkInternal(trc, reinterpret_cast<JSObject *>(thing));
231 23842393 : break;
232 : case JSTRACE_STRING:
233 3564439 : MarkInternal(trc, reinterpret_cast<JSString *>(thing));
234 3564439 : break;
235 : case JSTRACE_SCRIPT:
236 1719503 : MarkInternal(trc, static_cast<JSScript *>(thing));
237 1719503 : break;
238 : case JSTRACE_SHAPE:
239 1170716 : MarkInternal(trc, reinterpret_cast<Shape *>(thing));
240 1170716 : break;
241 : case JSTRACE_BASE_SHAPE:
242 140027 : MarkInternal(trc, reinterpret_cast<BaseShape *>(thing));
243 140027 : break;
244 : case JSTRACE_TYPE_OBJECT:
245 35495 : MarkInternal(trc, reinterpret_cast<types::TypeObject *>(thing));
246 35495 : break;
247 : #if JS_HAS_XML_SUPPORT
248 : case JSTRACE_XML:
249 45 : MarkInternal(trc, static_cast<JSXML *>(thing));
250 45 : break;
251 : #endif
252 : }
253 30472618 : }
254 :
255 : void
256 135081 : MarkGCThingRoot(JSTracer *trc, void *thing, const char *name)
257 : {
258 135081 : JS_ROOT_MARKING_ASSERT(trc);
259 135081 : JS_SET_TRACING_NAME(trc, name);
260 135081 : if (!thing)
261 0 : return;
262 135081 : MarkKind(trc, thing, GetGCThingTraceKind(thing));
263 : }
264 :
265 : /*** ID Marking ***/
266 :
267 : static inline void
268 4712038 : MarkIdInternal(JSTracer *trc, jsid *id)
269 : {
270 4712038 : if (JSID_IS_STRING(*id)) {
271 3770823 : JSString *str = JSID_TO_STRING(*id);
272 3770823 : MarkInternal(trc, str);
273 3770823 : *id = ATOM_TO_JSID(reinterpret_cast<JSAtom *>(str));
274 941215 : } else if (JS_UNLIKELY(JSID_IS_OBJECT(*id))) {
275 0 : JSObject *obj = JSID_TO_OBJECT(*id);
276 0 : MarkInternal(trc, obj);
277 0 : *id = OBJECT_TO_JSID(obj);
278 : }
279 4712038 : }
280 :
281 : void
282 4660197 : MarkId(JSTracer *trc, HeapId *id, const char *name)
283 : {
284 4660197 : JS_SET_TRACING_NAME(trc, name);
285 4660197 : MarkIdInternal(trc, id->unsafeGet());
286 4660197 : }
287 :
288 : void
289 0 : MarkIdRoot(JSTracer *trc, jsid *id, const char *name)
290 : {
291 0 : JS_ROOT_MARKING_ASSERT(trc);
292 0 : JS_SET_TRACING_NAME(trc, name);
293 0 : MarkIdInternal(trc, id);
294 0 : }
295 :
296 : void
297 0 : MarkIdRange(JSTracer *trc, size_t len, HeapId *vec, const char *name)
298 : {
299 0 : for (size_t i = 0; i < len; ++i) {
300 0 : JS_SET_TRACING_INDEX(trc, name, i);
301 0 : MarkIdInternal(trc, vec[i].unsafeGet());
302 : }
303 0 : }
304 :
305 : void
306 1597 : MarkIdRootRange(JSTracer *trc, size_t len, jsid *vec, const char *name)
307 : {
308 1597 : JS_ROOT_MARKING_ASSERT(trc);
309 53438 : for (size_t i = 0; i < len; ++i) {
310 51841 : JS_SET_TRACING_INDEX(trc, name, i);
311 51841 : MarkIdInternal(trc, &vec[i]);
312 : }
313 1597 : }
314 :
315 : /*** Value Marking ***/
316 :
317 : static inline void
318 30203592 : MarkValueInternal(JSTracer *trc, Value *v)
319 : {
320 30203592 : if (v->isMarkable()) {
321 10166770 : JS_ASSERT(v->toGCThing());
322 10166770 : return MarkKind(trc, v->toGCThing(), v->gcKind());
323 : }
324 : }
325 :
326 : void
327 15354 : MarkValue(JSTracer *trc, HeapValue *v, const char *name)
328 : {
329 15354 : JS_SET_TRACING_NAME(trc, name);
330 15354 : MarkValueInternal(trc, v->unsafeGet());
331 15354 : }
332 :
333 : void
334 61708 : MarkValueRoot(JSTracer *trc, Value *v, const char *name)
335 : {
336 61708 : JS_ROOT_MARKING_ASSERT(trc);
337 61708 : JS_SET_TRACING_NAME(trc, name);
338 61708 : MarkValueInternal(trc, v);
339 61708 : }
340 :
341 : void
342 8401974 : MarkValueRange(JSTracer *trc, size_t len, HeapValue *vec, const char *name)
343 : {
344 24904965 : for (size_t i = 0; i < len; ++i) {
345 16502991 : JS_SET_TRACING_INDEX(trc, name, i);
346 16502991 : MarkValueInternal(trc, vec[i].unsafeGet());
347 : }
348 8401974 : }
349 :
350 : void
351 647029 : MarkValueRootRange(JSTracer *trc, size_t len, Value *vec, const char *name)
352 : {
353 647029 : JS_ROOT_MARKING_ASSERT(trc);
354 3804037 : for (size_t i = 0; i < len; ++i) {
355 3157008 : JS_SET_TRACING_INDEX(trc, name, i);
356 3157008 : MarkValueInternal(trc, &vec[i]);
357 : }
358 647029 : }
359 :
360 : /*** Slot Marking ***/
361 :
362 : void
363 82975 : MarkSlot(JSTracer *trc, HeapSlot *s, const char *name)
364 : {
365 82975 : JS_SET_TRACING_NAME(trc, name);
366 82975 : MarkValueInternal(trc, s->unsafeGet());
367 82975 : }
368 :
369 : void
370 22601 : MarkArraySlots(JSTracer *trc, size_t len, HeapSlot *vec, const char *name)
371 : {
372 74564 : for (size_t i = 0; i < len; ++i) {
373 51963 : JS_SET_TRACING_INDEX(trc, name, i);
374 51963 : MarkValueInternal(trc, vec[i].unsafeGet());
375 : }
376 22601 : }
377 :
378 : void
379 3311434 : MarkObjectSlots(JSTracer *trc, JSObject *obj, uint32_t start, uint32_t nslots)
380 : {
381 3311434 : JS_ASSERT(obj->isNative());
382 13030963 : for (uint32_t i = start; i < (start + nslots); ++i) {
383 9719529 : JS_SET_TRACING_DETAILS(trc, js_PrintObjectSlotName, obj, i);
384 9719529 : MarkValueInternal(trc, obj->nativeGetSlotRef(i).unsafeGet());
385 : }
386 3311434 : }
387 :
388 : void
389 169534 : MarkCrossCompartmentSlot(JSTracer *trc, HeapSlot *s, const char *name)
390 : {
391 169534 : if (s->isMarkable()) {
392 83344 : Cell *cell = (Cell *)s->toGCThing();
393 83344 : JSRuntime *rt = trc->runtime;
394 83344 : if (rt->gcCurrentCompartment && cell->compartment() != rt->gcCurrentCompartment)
395 420 : return;
396 :
397 : /* In case we're called from a write barrier. */
398 82924 : if (rt->gcIncrementalCompartment && cell->compartment() != rt->gcIncrementalCompartment)
399 0 : return;
400 :
401 82924 : MarkSlot(trc, s, name);
402 : }
403 : }
404 :
405 : /*** Special Marking ***/
406 :
407 : void
408 100446 : MarkObject(JSTracer *trc, HeapPtr<GlobalObject, JSScript *> *thingp, const char *name)
409 : {
410 100446 : JS_SET_TRACING_NAME(trc, name);
411 100446 : MarkInternal(trc, thingp->get());
412 100446 : }
413 :
414 : void
415 612064 : MarkValueUnbarriered(JSTracer *trc, Value *v, const char *name)
416 : {
417 612064 : JS_SET_TRACING_NAME(trc, name);
418 612064 : MarkValueInternal(trc, v);
419 612064 : }
420 :
421 : /*** Push Mark Stack ***/
422 :
423 : #define JS_COMPARTMENT_ASSERT(rt, thing) \
424 : JS_ASSERT_IF((rt)->gcCurrentCompartment, \
425 : (thing)->compartment() == (rt)->gcCurrentCompartment);
426 :
427 : #define JS_COMPARTMENT_ASSERT_STR(rt, thing) \
428 : JS_ASSERT_IF((rt)->gcCurrentCompartment, \
429 : (thing)->compartment() == (rt)->gcCurrentCompartment || \
430 : (thing)->compartment() == (rt)->atomsCompartment);
431 :
432 : static void
433 2572 : PushMarkStack(GCMarker *gcmarker, JSXML *thing)
434 : {
435 2572 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
436 :
437 2572 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
438 2529 : gcmarker->pushXML(thing);
439 2572 : }
440 :
441 : static void
442 51869759 : PushMarkStack(GCMarker *gcmarker, JSObject *thing)
443 : {
444 51869759 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
445 :
446 51869759 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
447 12287237 : gcmarker->pushObject(thing);
448 51869759 : }
449 :
450 : static void
451 7679101 : PushMarkStack(GCMarker *gcmarker, JSFunction *thing)
452 : {
453 7679101 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
454 :
455 7679101 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
456 4951572 : gcmarker->pushObject(thing);
457 7679101 : }
458 :
459 : static void
460 54348483 : PushMarkStack(GCMarker *gcmarker, types::TypeObject *thing)
461 : {
462 54348483 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
463 :
464 54348483 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
465 2105262 : gcmarker->pushType(thing);
466 54348483 : }
467 :
468 : static void
469 : MarkChildren(JSTracer *trc, JSScript *script);
470 :
471 : static void
472 15240210 : PushMarkStack(GCMarker *gcmarker, JSScript *thing)
473 : {
474 15240210 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
475 :
476 : /*
477 : * We mark scripts directly rather than pushing on the stack as they can
478 : * refer to other scripts only indirectly (like via nested functions) and
479 : * we cannot get to deep recursion.
480 : */
481 15240210 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
482 6777038 : MarkChildren(gcmarker, thing);
483 15240210 : }
484 :
485 : static void
486 : ScanShape(GCMarker *gcmarker, Shape *shape);
487 :
488 : static void
489 62014394 : PushMarkStack(GCMarker *gcmarker, Shape *thing)
490 : {
491 62014394 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
492 :
493 : /* We mark shapes directly rather than pushing on the stack. */
494 62014394 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
495 12872354 : ScanShape(gcmarker, thing);
496 62014394 : }
497 :
498 : static inline void
499 : ScanBaseShape(GCMarker *gcmarker, BaseShape *base);
500 :
501 : static void
502 91843090 : PushMarkStack(GCMarker *gcmarker, BaseShape *thing)
503 : {
504 91843090 : JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing);
505 :
506 : /* We mark base shapes directly rather than pushing on the stack. */
507 91843090 : if (thing->markIfUnmarked(gcmarker->getMarkColor()))
508 14788220 : ScanBaseShape(gcmarker, thing);
509 91843090 : }
510 :
511 : static void
512 91712718 : ScanShape(GCMarker *gcmarker, Shape *shape)
513 : {
514 : restart:
515 91712718 : PushMarkStack(gcmarker, shape->base());
516 :
517 91712718 : const HeapId &id = shape->propidRef();
518 91712718 : if (JSID_IS_STRING(id))
519 61193841 : PushMarkStack(gcmarker, JSID_TO_STRING(id));
520 30518877 : else if (JS_UNLIKELY(JSID_IS_OBJECT(id)))
521 0 : PushMarkStack(gcmarker, JSID_TO_OBJECT(id));
522 :
523 91712718 : shape = shape->previous();
524 91712718 : if (shape && shape->markIfUnmarked(gcmarker->getMarkColor()))
525 78840364 : goto restart;
526 12872354 : }
527 :
528 : static inline void
529 14788220 : ScanBaseShape(GCMarker *gcmarker, BaseShape *base)
530 : {
531 14788220 : base->assertConsistency();
532 :
533 14788220 : if (base->hasGetterObject())
534 3384467 : PushMarkStack(gcmarker, base->getterObject());
535 :
536 14788220 : if (base->hasSetterObject())
537 527600 : PushMarkStack(gcmarker, base->setterObject());
538 :
539 14788220 : if (JSObject *parent = base->getObjectParent())
540 14262217 : PushMarkStack(gcmarker, parent);
541 :
542 : /*
543 : * All children of the owned base shape are consistent with its
544 : * unowned one, thus we do not need to trace through children of the
545 : * unowned base shape.
546 : */
547 14788220 : if (base->isOwned()) {
548 1561708 : UnownedBaseShape *unowned = base->baseUnowned();
549 1561708 : JS_ASSERT(base->compartment() == unowned->compartment());
550 1561708 : unowned->markIfUnmarked(gcmarker->getMarkColor());
551 : }
552 14788220 : }
553 :
554 : static inline void
555 313783947 : ScanLinearString(GCMarker *gcmarker, JSLinearString *str)
556 : {
557 313783947 : JS_COMPARTMENT_ASSERT_STR(gcmarker->runtime, str);
558 313783947 : JS_ASSERT(str->isMarked());
559 :
560 : /*
561 : * Add extra asserts to confirm the static type to detect incorrect string
562 : * mutations.
563 : */
564 313783947 : JS_ASSERT(str->JSString::isLinear());
565 627640508 : while (str->isDependent()) {
566 496447 : str = str->asDependent().base();
567 496447 : JS_ASSERT(str->JSString::isLinear());
568 496447 : JS_COMPARTMENT_ASSERT_STR(gcmarker->runtime, str);
569 496447 : if (!str->markIfUnmarked())
570 423833 : break;
571 : }
572 313783947 : }
573 :
574 : /*
575 : * The function tries to scan the whole rope tree using the marking stack as
576 : * temporary storage. If that becomes full, the unscanned ropes are added to
577 : * the delayed marking list. When the function returns, the marking stack is
578 : * at the same depth as it was on entry. This way we avoid using tags when
579 : * pushing ropes to the stack as ropes never leaks to other users of the
580 : * stack. This also assumes that a rope can only point to other ropes or
581 : * linear strings, it cannot refer to GC things of other types.
582 : */
583 : static void
584 2901642 : ScanRope(GCMarker *gcmarker, JSRope *rope)
585 : {
586 2901642 : ptrdiff_t savedPos = gcmarker->stack.position();
587 10248400 : for (;;) {
588 13150042 : JS_ASSERT(GetGCThingTraceKind(rope) == JSTRACE_STRING);
589 13150042 : JS_ASSERT(rope->JSString::isRope());
590 13150042 : JS_COMPARTMENT_ASSERT_STR(gcmarker->runtime, rope);
591 13150042 : JS_ASSERT(rope->isMarked());
592 13150042 : JSRope *next = NULL;
593 :
594 13150042 : JSString *right = rope->rightChild();
595 13150042 : if (right->markIfUnmarked()) {
596 3701163 : if (right->isLinear())
597 51920 : ScanLinearString(gcmarker, &right->asLinear());
598 : else
599 3649243 : next = &right->asRope();
600 : }
601 :
602 13150042 : JSString *left = rope->leftChild();
603 13150042 : if (left->markIfUnmarked()) {
604 6603767 : if (left->isLinear()) {
605 4610 : ScanLinearString(gcmarker, &left->asLinear());
606 : } else {
607 : /*
608 : * When both children are ropes, set aside the right one to
609 : * scan it later.
610 : */
611 6599157 : if (next && !gcmarker->stack.push(reinterpret_cast<uintptr_t>(next)))
612 0 : gcmarker->delayMarkingChildren(next);
613 6599157 : next = &left->asRope();
614 : }
615 : }
616 13150042 : if (next) {
617 10248398 : rope = next;
618 2901644 : } else if (savedPos != gcmarker->stack.position()) {
619 2 : JS_ASSERT(savedPos < gcmarker->stack.position());
620 2 : rope = reinterpret_cast<JSRope *>(gcmarker->stack.pop());
621 : } else {
622 : break;
623 : }
624 : }
625 2901642 : JS_ASSERT(savedPos == gcmarker->stack.position());
626 2901642 : }
627 :
628 : static inline void
629 316629059 : ScanString(GCMarker *gcmarker, JSString *str)
630 : {
631 316629059 : if (str->isLinear())
632 313727417 : ScanLinearString(gcmarker, &str->asLinear());
633 : else
634 2901642 : ScanRope(gcmarker, &str->asRope());
635 316629059 : }
636 :
637 : static inline void
638 411381905 : PushMarkStack(GCMarker *gcmarker, JSString *str)
639 : {
640 411381905 : JS_COMPARTMENT_ASSERT_STR(gcmarker->runtime, str);
641 :
642 : /*
643 : * As string can only refer to other strings we fully scan its GC graph
644 : * using the explicit stack when navigating the rope tree to avoid
645 : * dealing with strings on the stack in drainMarkStack.
646 : */
647 411381905 : if (str->markIfUnmarked())
648 271811714 : ScanString(gcmarker, str);
649 411381905 : }
650 :
651 : void
652 3363421 : MarkChildren(JSTracer *trc, JSObject *obj)
653 : {
654 3363421 : obj->markChildren(trc);
655 3363421 : }
656 :
657 : static void
658 17275905 : MarkChildren(JSTracer *trc, JSString *str)
659 : {
660 17275905 : if (str->isDependent())
661 1019 : str->asDependent().markChildren(trc);
662 17274886 : else if (str->isRope())
663 2154 : str->asRope().markChildren(trc);
664 17275905 : }
665 :
666 : static void
667 6864592 : MarkChildren(JSTracer *trc, JSScript *script)
668 : {
669 6864592 : script->markChildren(trc);
670 6864592 : }
671 :
672 : static void
673 3699693 : MarkChildren(JSTracer *trc, Shape *shape)
674 : {
675 3699693 : shape->markChildren(trc);
676 3699693 : }
677 :
678 : static void
679 1144121 : MarkChildren(JSTracer *trc, BaseShape *base)
680 : {
681 1144121 : base->markChildren(trc);
682 1144121 : }
683 :
684 : /*
685 : * This function is used by the cycle collector to trace through the
686 : * children of a BaseShape (and its baseUnowned(), if any). The cycle
687 : * collector does not directly care about BaseShapes, so only the
688 : * getter, setter, and parent are marked. Furthermore, the parent is
689 : * marked only if it isn't the same as prevParent, which will be
690 : * updated to the current shape's parent.
691 : */
692 : inline void
693 958406 : MarkCycleCollectorChildren(JSTracer *trc, BaseShape *base, JSObject **prevParent)
694 : {
695 958406 : JS_ASSERT(base);
696 :
697 : /*
698 : * The cycle collector does not need to trace unowned base shapes,
699 : * as they have the same getter, setter and parent as the original
700 : * base shape.
701 : */
702 958406 : base->assertConsistency();
703 :
704 958406 : if (base->hasGetterObject()) {
705 12402 : JSObject *tmp = base->getterObject();
706 12402 : MarkObjectUnbarriered(trc, &tmp, "getter");
707 12402 : JS_ASSERT(tmp == base->getterObject());
708 : }
709 :
710 958406 : if (base->hasSetterObject()) {
711 471 : JSObject *tmp = base->setterObject();
712 471 : MarkObjectUnbarriered(trc, &tmp, "setter");
713 471 : JS_ASSERT(tmp == base->setterObject());
714 : }
715 :
716 958406 : JSObject *parent = base->getObjectParent();
717 958406 : if (parent && parent != *prevParent) {
718 330139 : MarkObjectUnbarriered(trc, &parent, "parent");
719 330139 : JS_ASSERT(parent == base->getObjectParent());
720 330139 : *prevParent = parent;
721 : }
722 958406 : }
723 :
724 : /*
725 : * This function is used by the cycle collector to trace through a
726 : * shape. The cycle collector does not care about shapes or base
727 : * shapes, so those are not marked. Instead, any shapes or base shapes
728 : * that are encountered have their children marked. Stack space is
729 : * bounded. If two shapes in a row have the same parent pointer, the
730 : * parent pointer will only be marked once.
731 : */
732 : void
733 396130 : MarkCycleCollectorChildren(JSTracer *trc, Shape *shape)
734 : {
735 396130 : JSObject *prevParent = NULL;
736 958406 : do {
737 958406 : MarkCycleCollectorChildren(trc, shape->base(), &prevParent);
738 958406 : MarkId(trc, &shape->propidRef(), "propid");
739 958406 : shape = shape->previous();
740 : } while (shape);
741 396130 : }
742 :
743 : static void
744 2104422 : ScanTypeObject(GCMarker *gcmarker, types::TypeObject *type)
745 : {
746 2104422 : if (!type->singleton) {
747 2002755 : unsigned count = type->getPropertyCount();
748 2009420 : for (unsigned i = 0; i < count; i++) {
749 6665 : types::Property *prop = type->getProperty(i);
750 6665 : if (prop && JSID_IS_STRING(prop->id))
751 3964 : PushMarkStack(gcmarker, JSID_TO_STRING(prop->id));
752 : }
753 : }
754 :
755 2104422 : if (type->proto)
756 2041397 : PushMarkStack(gcmarker, type->proto);
757 :
758 2104422 : if (type->newScript) {
759 9 : PushMarkStack(gcmarker, type->newScript->fun);
760 9 : PushMarkStack(gcmarker, type->newScript->shape);
761 : }
762 :
763 2104422 : if (type->interpretedFunction)
764 179593 : PushMarkStack(gcmarker, type->interpretedFunction);
765 :
766 2104422 : if (type->singleton && !type->lazy())
767 59810 : PushMarkStack(gcmarker, type->singleton);
768 :
769 2104422 : if (type->interpretedFunction)
770 179593 : PushMarkStack(gcmarker, type->interpretedFunction);
771 2104422 : }
772 :
773 : static void
774 425099 : MarkChildren(JSTracer *trc, types::TypeObject *type)
775 : {
776 425099 : if (!type->singleton) {
777 414207 : unsigned count = type->getPropertyCount();
778 416287 : for (unsigned i = 0; i < count; i++) {
779 2080 : types::Property *prop = type->getProperty(i);
780 2080 : if (prop)
781 2080 : MarkId(trc, &prop->id, "type_prop");
782 : }
783 : }
784 :
785 425099 : if (type->proto)
786 410704 : MarkObject(trc, &type->proto, "type_proto");
787 :
788 425099 : if (type->singleton && !type->lazy())
789 7179 : MarkObject(trc, &type->singleton, "type_singleton");
790 :
791 425099 : if (type->newScript) {
792 124 : MarkObject(trc, &type->newScript->fun, "type_new_function");
793 124 : MarkShape(trc, &type->newScript->shape, "type_new_shape");
794 : }
795 :
796 425099 : if (type->interpretedFunction)
797 35673 : MarkObject(trc, &type->interpretedFunction, "type_function");
798 425099 : }
799 :
800 : #ifdef JS_HAS_XML_SUPPORT
801 : static void
802 2529 : MarkChildren(JSTracer *trc, JSXML *xml)
803 : {
804 2529 : js_TraceXML(trc, xml);
805 2529 : }
806 : #endif
807 :
808 : template<typename T>
809 : void
810 0 : PushArenaTyped(GCMarker *gcmarker, ArenaHeader *aheader)
811 : {
812 0 : for (CellIterUnderGC i(aheader); !i.done(); i.next())
813 0 : PushMarkStack(gcmarker, i.get<T>());
814 0 : }
815 :
816 : void
817 0 : PushArena(GCMarker *gcmarker, ArenaHeader *aheader)
818 : {
819 0 : switch (MapAllocToTraceKind(aheader->getAllocKind())) {
820 : case JSTRACE_OBJECT:
821 0 : PushArenaTyped<JSObject>(gcmarker, aheader);
822 0 : break;
823 :
824 : case JSTRACE_STRING:
825 0 : PushArenaTyped<JSString>(gcmarker, aheader);
826 0 : break;
827 :
828 : case JSTRACE_SCRIPT:
829 0 : PushArenaTyped<JSScript>(gcmarker, aheader);
830 0 : break;
831 :
832 : case JSTRACE_SHAPE:
833 0 : PushArenaTyped<js::Shape>(gcmarker, aheader);
834 0 : break;
835 :
836 : case JSTRACE_BASE_SHAPE:
837 0 : PushArenaTyped<js::BaseShape>(gcmarker, aheader);
838 0 : break;
839 :
840 : case JSTRACE_TYPE_OBJECT:
841 0 : PushArenaTyped<js::types::TypeObject>(gcmarker, aheader);
842 0 : break;
843 :
844 : #if JS_HAS_XML_SUPPORT
845 : case JSTRACE_XML:
846 0 : PushArenaTyped<JSXML>(gcmarker, aheader);
847 0 : break;
848 : #endif
849 : }
850 0 : }
851 :
852 : } /* namespace gc */
853 :
854 : using namespace js::gc;
855 :
856 : struct SlotArrayLayout
857 : {
858 : union {
859 : HeapSlot *end;
860 : js::Class *clasp;
861 : };
862 : union {
863 : HeapSlot *start;
864 : uintptr_t index;
865 : };
866 : JSObject *obj;
867 :
868 : static void staticAsserts() {
869 : /* This should have the same layout as three mark stack items. */
870 : JS_STATIC_ASSERT(sizeof(SlotArrayLayout) == 3 * sizeof(uintptr_t));
871 : }
872 : };
873 :
874 : /*
875 : * During incremental GC, we return from drainMarkStack without having processed
876 : * the entire stack. At that point, JS code can run and reallocate slot arrays
877 : * that are stored on the stack. To prevent this from happening, we replace all
878 : * ValueArrayTag stack items with SavedValueArrayTag. In the latter, slots
879 : * pointers are replaced with slot indexes.
880 : *
881 : * We also replace the slot array end pointer (which can be derived from the obj
882 : * pointer) with the object's class. During JS executation, array slowification
883 : * can cause the layout of slots to change. We can observe that slowification
884 : * happened if the class changed; in that case, we completely rescan the array.
885 : */
886 : void
887 0 : GCMarker::saveValueRanges()
888 : {
889 0 : for (uintptr_t *p = stack.tos; p > stack.stack; ) {
890 0 : uintptr_t tag = *--p & StackTagMask;
891 0 : if (tag == ValueArrayTag) {
892 0 : p -= 2;
893 0 : SlotArrayLayout *arr = reinterpret_cast<SlotArrayLayout *>(p);
894 0 : JSObject *obj = arr->obj;
895 :
896 0 : if (obj->getClass() == &ArrayClass) {
897 0 : HeapSlot *vp = obj->getDenseArrayElements();
898 0 : JS_ASSERT(arr->start >= vp &&
899 0 : arr->end == vp + obj->getDenseArrayInitializedLength());
900 0 : arr->index = arr->start - vp;
901 : } else {
902 0 : HeapSlot *vp = obj->fixedSlots();
903 0 : unsigned nfixed = obj->numFixedSlots();
904 0 : if (arr->start >= vp && arr->start < vp + nfixed) {
905 0 : JS_ASSERT(arr->end == vp + Min(nfixed, obj->slotSpan()));
906 0 : arr->index = arr->start - vp;
907 : } else {
908 0 : JS_ASSERT(arr->start >= obj->slots &&
909 0 : arr->end == obj->slots + obj->slotSpan() - nfixed);
910 0 : arr->index = (arr->start - obj->slots) + nfixed;
911 : }
912 : }
913 0 : arr->clasp = obj->getClass();
914 0 : p[2] |= SavedValueArrayTag;
915 0 : } else if (tag == SavedValueArrayTag) {
916 0 : p -= 2;
917 : }
918 : }
919 0 : }
920 :
921 : bool
922 0 : GCMarker::restoreValueArray(JSObject *obj, void **vpp, void **endp)
923 : {
924 0 : uintptr_t start = stack.pop();
925 0 : js::Class *clasp = reinterpret_cast<js::Class *>(stack.pop());
926 :
927 0 : JS_ASSERT(obj->getClass() == clasp ||
928 0 : (clasp == &ArrayClass && obj->getClass() == &SlowArrayClass));
929 :
930 0 : if (clasp == &ArrayClass) {
931 0 : if (obj->getClass() != &ArrayClass)
932 0 : return false;
933 :
934 0 : uint32_t initlen = obj->getDenseArrayInitializedLength();
935 0 : HeapSlot *vp = obj->getDenseArrayElements();
936 0 : if (start < initlen) {
937 0 : *vpp = vp + start;
938 0 : *endp = vp + initlen;
939 : } else {
940 : /* The object shrunk, in which case no scanning is needed. */
941 0 : *vpp = *endp = vp;
942 : }
943 : } else {
944 0 : HeapSlot *vp = obj->fixedSlots();
945 0 : unsigned nfixed = obj->numFixedSlots();
946 0 : unsigned nslots = obj->slotSpan();
947 0 : if (start < nfixed) {
948 0 : *vpp = vp + start;
949 0 : *endp = vp + Min(nfixed, nslots);
950 0 : } else if (start < nslots) {
951 0 : *vpp = obj->slots + start - nfixed;
952 0 : *endp = obj->slots + nslots - nfixed;
953 : } else {
954 : /* The object shrunk, in which case no scanning is needed. */
955 0 : *vpp = *endp = obj->slots;
956 : }
957 : }
958 :
959 0 : JS_ASSERT(*vpp <= *endp);
960 0 : return true;
961 : }
962 :
963 : inline void
964 54576924 : GCMarker::processMarkStackTop(SliceBudget &budget)
965 : {
966 : /*
967 : * The function uses explicit goto and implements the scanning of the
968 : * object directly. It allows to eliminate the tail recursion and
969 : * significantly improve the marking performance, see bug 641025.
970 : */
971 : HeapSlot *vp, *end;
972 : JSObject *obj;
973 :
974 54576924 : uintptr_t addr = stack.pop();
975 54576924 : uintptr_t tag = addr & StackTagMask;
976 54576924 : addr &= ~StackTagMask;
977 :
978 54576924 : if (tag == ValueArrayTag) {
979 : JS_STATIC_ASSERT(ValueArrayTag == 0);
980 35234340 : JS_ASSERT(!(addr & Cell::CellMask));
981 35234340 : obj = reinterpret_cast<JSObject *>(addr);
982 35234340 : uintptr_t addr2 = stack.pop();
983 35234340 : uintptr_t addr3 = stack.pop();
984 35234340 : JS_ASSERT(addr2 <= addr3);
985 35234340 : JS_ASSERT((addr3 - addr2) % sizeof(Value) == 0);
986 35234340 : vp = reinterpret_cast<HeapSlot *>(addr2);
987 35234340 : end = reinterpret_cast<HeapSlot *>(addr3);
988 35234340 : goto scan_value_array;
989 : }
990 :
991 19342584 : if (tag == ObjectTag) {
992 17235633 : obj = reinterpret_cast<JSObject *>(addr);
993 17235633 : JS_COMPARTMENT_ASSERT(runtime, obj);
994 17235633 : goto scan_obj;
995 : }
996 :
997 2106951 : if (tag == TypeTag) {
998 2104422 : ScanTypeObject(this, reinterpret_cast<types::TypeObject *>(addr));
999 2529 : } else if (tag == SavedValueArrayTag) {
1000 0 : JS_ASSERT(!(addr & Cell::CellMask));
1001 0 : obj = reinterpret_cast<JSObject *>(addr);
1002 0 : if (restoreValueArray(obj, (void **)&vp, (void **)&end))
1003 0 : goto scan_value_array;
1004 : else
1005 0 : goto scan_obj;
1006 : } else {
1007 2529 : JS_ASSERT(tag == XmlTag);
1008 2529 : MarkChildren(this, reinterpret_cast<JSXML *>(addr));
1009 : }
1010 2106951 : budget.step();
1011 2106951 : return;
1012 :
1013 : scan_value_array:
1014 89523945 : JS_ASSERT(vp <= end);
1015 353910578 : while (vp != end) {
1016 211925000 : budget.step();
1017 211925000 : if (budget.isOverBudget()) {
1018 0 : pushValueArray(obj, vp, end);
1019 0 : return;
1020 : }
1021 :
1022 211925000 : const Value &v = *vp++;
1023 211925000 : if (v.isString()) {
1024 84722295 : JSString *str = v.toString();
1025 84722295 : JS_COMPARTMENT_ASSERT_STR(runtime, str);
1026 84722295 : if (str->markIfUnmarked())
1027 44817345 : ScanString(this, str);
1028 127202705 : } else if (v.isObject()) {
1029 47976057 : JSObject *obj2 = &v.toObject();
1030 47976057 : JS_COMPARTMENT_ASSERT(runtime, obj2);
1031 47976057 : if (obj2->markIfUnmarked(getMarkColor())) {
1032 37062312 : pushValueArray(obj, vp, end);
1033 37062312 : obj = obj2;
1034 37062312 : goto scan_obj;
1035 : }
1036 : }
1037 : }
1038 52461633 : return;
1039 :
1040 : scan_obj:
1041 : {
1042 54297945 : JS_COMPARTMENT_ASSERT(runtime, obj);
1043 :
1044 54297945 : budget.step();
1045 54297945 : if (budget.isOverBudget()) {
1046 0 : pushObject(obj);
1047 0 : return;
1048 : }
1049 :
1050 54297945 : types::TypeObject *type = obj->typeFromGC();
1051 54297945 : PushMarkStack(this, type);
1052 :
1053 54297945 : Shape *shape = obj->lastProperty();
1054 54297945 : PushMarkStack(this, shape);
1055 :
1056 : /* Call the trace hook if necessary. */
1057 54297945 : Class *clasp = shape->getObjectClass();
1058 54297945 : if (clasp->trace) {
1059 47435560 : if (clasp == &ArrayClass) {
1060 1328001 : JS_ASSERT(!shape->isNative());
1061 1328001 : vp = obj->getDenseArrayElements();
1062 1328001 : end = vp + obj->getDenseArrayInitializedLength();
1063 1328001 : goto scan_value_array;
1064 : } else {
1065 0 : JS_ASSERT_IF(runtime->gcIncrementalState != NO_INCREMENTAL,
1066 46107559 : clasp->flags & JSCLASS_IMPLEMENTS_BARRIERS);
1067 : }
1068 46107559 : clasp->trace(this, obj);
1069 : }
1070 :
1071 52969944 : if (!shape->isNative())
1072 8340 : return;
1073 :
1074 52961604 : unsigned nslots = obj->slotSpan();
1075 52961604 : vp = obj->fixedSlots();
1076 52961604 : if (obj->slots) {
1077 9039797 : unsigned nfixed = obj->numFixedSlots();
1078 9039797 : if (nslots > nfixed) {
1079 9039797 : pushValueArray(obj, vp, vp + nfixed);
1080 9039797 : vp = obj->slots;
1081 9039797 : end = vp + (nslots - nfixed);
1082 9039797 : goto scan_value_array;
1083 : }
1084 : }
1085 43921807 : JS_ASSERT(nslots <= obj->numFixedSlots());
1086 43921807 : end = vp + nslots;
1087 43921807 : goto scan_value_array;
1088 : }
1089 : }
1090 :
1091 : bool
1092 101970 : GCMarker::drainMarkStack(SliceBudget &budget)
1093 : {
1094 : #ifdef DEBUG
1095 101970 : JSRuntime *rt = runtime;
1096 :
1097 : struct AutoCheckCompartment {
1098 : JSRuntime *runtime;
1099 101970 : AutoCheckCompartment(JSRuntime *rt) : runtime(rt) {
1100 101970 : runtime->gcCheckCompartment = runtime->gcCurrentCompartment;
1101 101970 : }
1102 101970 : ~AutoCheckCompartment() { runtime->gcCheckCompartment = NULL; }
1103 203940 : } acc(rt);
1104 : #endif
1105 :
1106 101970 : if (budget.isOverBudget())
1107 0 : return false;
1108 :
1109 0 : for (;;) {
1110 54780864 : while (!stack.isEmpty()) {
1111 54576924 : processMarkStackTop(budget);
1112 54576924 : if (budget.isOverBudget()) {
1113 0 : saveValueRanges();
1114 0 : return false;
1115 : }
1116 : }
1117 :
1118 101970 : if (!hasDelayedChildren())
1119 : break;
1120 :
1121 : /*
1122 : * Mark children of things that caused too deep recursion during the
1123 : * above tracing. Don't do this until we're done with everything
1124 : * else.
1125 : */
1126 0 : if (!markDelayedChildren(budget)) {
1127 0 : saveValueRanges();
1128 0 : return false;
1129 : }
1130 : }
1131 :
1132 101970 : return true;
1133 : }
1134 :
1135 : void
1136 25995793 : TraceChildren(JSTracer *trc, void *thing, JSGCTraceKind kind)
1137 : {
1138 25995793 : switch (kind) {
1139 : case JSTRACE_OBJECT:
1140 3363421 : MarkChildren(trc, static_cast<JSObject *>(thing));
1141 3363421 : break;
1142 :
1143 : case JSTRACE_STRING:
1144 17275905 : MarkChildren(trc, static_cast<JSString *>(thing));
1145 17275905 : break;
1146 :
1147 : case JSTRACE_SCRIPT:
1148 87554 : MarkChildren(trc, static_cast<JSScript *>(thing));
1149 87554 : break;
1150 :
1151 : case JSTRACE_SHAPE:
1152 3699693 : MarkChildren(trc, static_cast<Shape *>(thing));
1153 3699693 : break;
1154 :
1155 : case JSTRACE_BASE_SHAPE:
1156 1144121 : MarkChildren(trc, static_cast<BaseShape *>(thing));
1157 1144121 : break;
1158 :
1159 : case JSTRACE_TYPE_OBJECT:
1160 425099 : MarkChildren(trc, (types::TypeObject *)thing);
1161 425099 : break;
1162 :
1163 : #if JS_HAS_XML_SUPPORT
1164 : case JSTRACE_XML:
1165 0 : MarkChildren(trc, static_cast<JSXML *>(thing));
1166 0 : break;
1167 : #endif
1168 : }
1169 25995793 : }
1170 :
1171 : void
1172 8656275 : CallTracer(JSTracer *trc, void *thing, JSGCTraceKind kind)
1173 : {
1174 8656275 : JS_ASSERT(thing);
1175 8656275 : MarkKind(trc, thing, kind);
1176 8656275 : }
1177 :
1178 : } /* namespace js */
|