/*!
 * \file overrideFunc.S
 * \brief This file is used to override JVM inner function.<br>
 * Copyright (C) 2011-2015 Nippon Telegraph and Telephone Corporation
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 *
 */

/* Override functions of "do_oop" families */
	.file "overrideFunc.S"

#define OVERRIDE_FUNC_DEFINE(header, ary_idx) \
		.global header##_override_func_##ary_idx ;         \
		.type header##_override_func_##ary_idx, @function; \
	header##_override_func_##ary_idx: ;

#if defined(__amd64__)
	
	/* Macro for function parts define. */
	
	#define DO_SAVE_REG \
		sub $24, %rsp; \
		mov %rdx, 16(%rsp); /* 3rd argument */ \
		mov %rsi,  8(%rsp); /* 2nd argument */ \
		mov %rdi,   (%rsp); /* 1st argument */
	
	#define DO_LOAD_REG \
		mov   (%rsp), %rdi; /* 1st argument */ \
		mov  8(%rsp), %rsi; /* 2nd argument */ \
		mov 16(%rsp), %rdx; /* 3rd argument */ \
		add $24, %rsp;
	
	#define GET_HOOK_REC(header, ary_idx) \
		/* get and set override info */ ; \
		mov header##_hook@GOTPCREL(%rip), %r11; \
		lea ary_idx*64(%r11), %r11; /* 64=sizeof(THookFunctionInfo) */
	
	#define GET_SOURCE_FUNC(header, ary_idx) \
		GET_HOOK_REC(header, ary_idx)                            ; \
		mov 32(%r11), %rax; /* Original vfunc offset in entry. */ ; \
		push %rax;
	
	#define GET_ALL_FUNC_ADDR(header, ary_idx) \
		GET_HOOK_REC(header, ary_idx)                        ; \
		sub $16, %rsp; \
		movdqa 32(%r11), %xmm0; /* Copy Original&After func to XMM */ \
		movdqu %xmm0, (%rsp); /* Store Original&After func to stack */
	
	#define DO_CHECK_PERM(oop_reg_ofs) \
		mov is_in_permanent@GOTPCREL(%rip), %r11; \
		mov collectedHeap@GOTPCREL(%rip), %rdi;   \
		mov (%rdi), %rdi;                         \
		mov oop_reg_ofs(%rsp), %rsi;              \
		call *(%r11);                             \
		test %al, %al;                            \
	
	#define DO_JMP_OVERRIDE(func, oop_reg_ofs) \
		mov oop_reg_ofs(%rsp), %rdi;     \
		mov func##@GOTPCREL(%rip), %r11; \
		call *%r11;
	
	#define DO_JMP_ANY_FUNC(nextFunc) \
		pop %r11;        \
		test %r11, %r11; \
		jz nextFunc;     \
		call *%r11;
	
	/* Macro for override function define. */
	
	#define OVERRIDE_DEFINE(header, ary_idx, func, oop_reg_idx) \
		OVERRIDE_FUNC_DEFINE(header, ary_idx) \
		                                      \
		/* Save argument registers */         \
		DO_SAVE_REG                           \
		                                      \
		/* Get and set override info. */ ;    \
		GET_SOURCE_FUNC(header, ary_idx)      \
		                                      \
		/* Check permanent object. */ ;       \
		DO_CHECK_PERM(oop_reg_idx*8)          \
		jne .LORIGINAL_FUNC;                  \
		                                      \
		/* Call callback function. */ ;       \
		DO_JMP_OVERRIDE(func, oop_reg_idx*8)  \
		jmp .LORIGINAL_FUNC

	#ifdef AVX
	  #define CHECK_VTABLE(oop_reg, valid_caller) \
	    vmovq (oop_reg), %xmm0;                  \
	    vshufpd $0, %xmm0, %xmm1, %xmm1;         \
	    mov valid_caller##@GOTPCREL(%rip), %r11; \
	    vpcmpeqq (%r11), %xmm1, %xmm1;           \
	    vptest %xmm1, %xmm1;
	#elif defined SSE4_1
	  #define CHECK_VTABLE(oop_reg, valid_caller) \
	    movq (oop_reg), %xmm0;                   \
	    shufpd $0, %xmm0, %xmm1;                 \
	    mov valid_caller##@GOTPCREL(%rip), %r11; \
	    pcmpeqq (%r11), %xmm1;                   \
	    ptest %xmm1, %xmm1;
	#else
	  #define CHECK_VTABLE(oop_reg, valid_caller) \
	    mov (oop_reg), oop_reg;                  \
	    xor %rax, %rax;                          \
	    mov valid_caller##@GOTPCREL(%rip), %r11; \
	    cmp (%r11), oop_reg;                     \
	    cmove %r11, %rax;                        \
	    cmp 8(%r11), oop_reg;                    \
	    cmove %r11, %rax;                        \
	    test %rax, %rax;
	#endif
	
	#define OVERRIDE_DEFINE_WITH_CALLER_CHECK(header, ary_idx, func, oop_reg_idx, valid_caller) \
		OVERRIDE_FUNC_DEFINE(header, ary_idx)    \
		                                         \
		/* Save argument registers */            \
		DO_SAVE_REG                              \
		                                         \
		/* Get and set override info. */ ;       \
		GET_SOURCE_FUNC(header, ary_idx)         \
		                                         \
		/* Check vtable of caller */             \
		CHECK_VTABLE(%rdx, valid_caller);        \
		jz .LORIGINAL_FUNC;                      \
		                                         \
		/* Check permanent object. */ ;          \
		DO_CHECK_PERM(oop_reg_idx*8)             \
		jne .LORIGINAL_FUNC;                     \
		                                         \
		/* Call callback function. */ ;          \
		DO_JMP_OVERRIDE(func, oop_reg_idx*8)     \
		jmp .LORIGINAL_FUNC

	#define OVERRIDE_ADJ_DEFINE(header, ary_idx, func, oop_reg_idx) \
		OVERRIDE_FUNC_DEFINE(header, ary_idx) \
		                                      \
		/* Save argument registers */         \
		DO_SAVE_REG                           \
		                                      \
		/* Get and set override info. */ ;    \
		GET_SOURCE_FUNC(header, ary_idx)      \
		                                      \
		/* Check permanent object. */ ;       \
		DO_CHECK_PERM(oop_reg_idx*8)          \
		jz .LORIGINAL_FUNC;                   \
		                                      \
		/* Call callback function. */ ;       \
		DO_JMP_OVERRIDE(func, oop_reg_idx*8)  \
		jmp .LORIGINAL_FUNC
	
	#define OVERRIDE_CALLBACK_DEFINE(header, ary_idx, func) \
		OVERRIDE_FUNC_DEFINE(header, ary_idx)      \
		                                           \
		/* Get all override info. */ ;             \
		GET_ALL_FUNC_ADDR(header, ary_idx)         \
		                                           \
		/* Save argument registers */              \
		DO_SAVE_REG                                \
		                                           \
		/* Call before callback with 2nd arg. */ ; \
		DO_JMP_OVERRIDE(func, 8)                   \
		                                           \
		                                           \
		/* call callback target */                 \
		DO_LOAD_REG                                \
		pop %r11;                                  \
		callq *%r11;                               \
		                                           \
		/* After Callback */                       \
		DO_JMP_ANY_FUNC(.RETURN_CALLER_FUNC)       \
		                                           \
		/* Return to caller */                     \
		jmp .RETURN_CALLER_FUNC
	
	/* Macro for hook each target function. */
	
	#define OVERRIDE_DO_BLK_CAREFUL(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForSweep, 2)
	#define OVERRIDE_OOP_FOLLOW_CONTENTS(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForParallel, 2)
	#define OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForParOld, 3)
	#define OVERRIDE_OOP_OOP_ITERATE(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForIterate, 2)
	#define OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(header, ary_idx, valid_caller) \
		OVERRIDE_DEFINE_WITH_CALLER_CHECK(header, ary_idx, callbackForIterate, 2, valid_caller)
	
	#define OVERRIDE_OOP_PAR_ADJUST(header, ary_idx) \
		OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForAdjustPtr, 2)
	#define OVERRIDE_OOP_DO_ADDR(header, ary_idx) \
		OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForDoAddr, 2)
	#define OVERRIDE_OOP_UPDATE_PTR(header, ary_idx) \
		OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForUpdatePtr, 3)
	
	#define OVERRIDE_JVMTI_ITERATE(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForJvmtiIterate, 2)
	
#elif defined(__i386__)
	
	#ifdef __i686
		/*
		 * This statement is for avoiding illegal conversion by preprocceor.
		 * before : "call __i686.get_pc_thunk.bx;"
		 * after  : "call 1.get_pc_thunk.bx;"
		 */
		#undef __i686
		#define AVOID__i686
	#endif
	
	/* Macro for function parts define. */
	
        #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 6)
                #define GET_GOT_ADDR \
                        push %ebx;                         \
                        call __x86.get_pc_thunk.bx;       \
                        addl $_GLOBAL_OFFSET_TABLE_, %ebx;
        #else
                #define GET_GOT_ADDR \
                        push %ebx;                         \
                        call __i686.get_pc_thunk.bx;       \
                        addl $_GLOBAL_OFFSET_TABLE_, %ebx;
        #endif
	
	#define GET_HOOK_REC(header, ary_idx)      \
		mov header##_hook@GOT(%ebx), %ecx; \
		lea ary_idx*28(%ecx), %ecx; /* 28=sizeof(THookFunctionInfo) */

	#define GET_SOURCE_FUNC(header, ary_idx) \
		GET_HOOK_REC(header, ary_idx)                                 \
		mov 16(%ecx), %eax;    /* original hook offset in entry. */ ; \
		push %eax;
	
	#define DO_CHECK_PERM(oop_reg) \
		mov (oop_reg)+12(%esp), %edx;           \
		mov is_in_permanent@GOTOFF(%ebx), %ecx; \
		mov collectedHeap@GOTOFF(%ebx), %eax;   \
		push %edx;                              \
		push %eax;                              \
		call *%ecx;                             \
		                                        \
		/* We recover 1 stack only.             \
		   Others (oop), we will recover in .LORIGINAL_FUNC . */ \
		addl $0x4, %esp;                        \
		                                        \
		test %al, %al;
	
	#define DO_JMP_OVERRIDE(func) \
		mov func##@GOT(%ebx), %edx; \
		call *%edx;

	#define GET_NEXT_JMP_FUNC(header, ary_idx, offset) \
	        GET_HOOK_REC(header, ary_idx) \
		mov offset(%ecx), %edx;       \
		test %edx, %edx;
	
	/* Macro for override function define. */
	
	#define OVERRIDE_DEFINE(header, ary_idx, func, oop_reg) \
		OVERRIDE_FUNC_DEFINE(header, ary_idx) \
		                                      \
		/* Get GOT base address. */ ;         \
		GET_GOT_ADDR                          \
		                                      \
		/* get and set override info */ ;     \
		GET_SOURCE_FUNC(header, ary_idx)      \
		                                      \
		/* Check permanent object. */ ;       \
		DO_CHECK_PERM(oop_reg)                \
		jne .LORIGINAL_FUNC;                  \
		                                      \
		/* Call callback function. */ ;       \
		DO_JMP_OVERRIDE(func)                 \
		jmp .LORIGINAL_FUNC;

	#ifdef AVX
	  #define CHECK_VTABLE(oop_reg, valid_caller) \
            vmovd (oop_reg), %xmm0;             \
	    vpshufd $0, %xmm0, %xmm1;           \
	    mov valid_caller##@GOT(%ebx), %ecx; \
	    vmovq (%ecx), %xmm1;                \
	    vpcmpeqd %xmm0, %xmm1, %xmm1;       \
	    vptest %xmm1, %xmm1;
	#elif defined SSE4_1
	  #define CHECK_VTABLE(oop_reg, valid_caller) \
            movd (oop_reg), %xmm0;              \
	    pshufd $0, %xmm0, %xmm1;            \
	    mov valid_caller##@GOT(%ebx), %ecx; \
	    movq (%ecx), %xmm1;                 \
	    pcmpeqd %xmm0, %xmm1;               \
	    ptest %xmm1, %xmm1;
	#else
	  #define CHECK_VTABLE(oop_reg, valid_caller) \
	    mov (oop_reg), oop_reg;             \
	    xor %edx, %edx;                     \
	    mov valid_caller##@GOT(%ebx), %ecx; \
	    cmp (%ecx), oop_reg;                \
	    cmove %ecx, %edx;                   \
	    cmp 4(%ecx), oop_reg;               \
	    cmove %ecx, %edx;                   \
	    test %edx, %edx;
	#endif

	#define OVERRIDE_DEFINE_WITH_CALLER_CHECK(header, ary_idx, func, oop_reg, valid_caller) \
		OVERRIDE_FUNC_DEFINE(header, ary_idx) \
		                                      \
		/* Get GOT base address. */ ;         \
		GET_GOT_ADDR                          \
		                                      \
		/* get and set override info */ ;     \
		GET_SOURCE_FUNC(header, ary_idx)      \
		                                      \
		/* Get 3rd arg (OopClosure) */        \
		mov 20(%esp), %eax;                   \
		                                      \
		/* Dummy stack for jmp */             \
		subl $4, %esp;                        \
		/* Check vtable of caller */          \
		CHECK_VTABLE(%eax, valid_caller);     \
		jz .LORIGINAL_FUNC;                   \
		addl $4, %esp;                        \
		                                      \
		/* Check permanent object. */ ;       \
		DO_CHECK_PERM(oop_reg)                \
		jne .LORIGINAL_FUNC;                  \
		                                      \
		/* Call callback function. */ ;       \
		DO_JMP_OVERRIDE(func)                 \
		jmp .LORIGINAL_FUNC;
	
	#define OVERRIDE_ADJ_DEFINE(header, ary_idx, func, oop_reg) \
		OVERRIDE_FUNC_DEFINE(header, ary_idx) \
		                                      \
		/* Get GOT base address. */ ;         \
		GET_GOT_ADDR                          \
		                                      \
		/* get and set override info */ ;     \
		GET_SOURCE_FUNC(header, ary_idx)      \
		                                      \
		/* Check permanent object. */ ;       \
		DO_CHECK_PERM(oop_reg)                \
		jz .LORIGINAL_FUNC;                   \
		                                      \
		/* Call callback function. */ ;       \
		DO_JMP_OVERRIDE(func)                 \
		jmp .LORIGINAL_FUNC;

	#ifdef AVX
	  #define REPUSH_ARGS \
	    vmovq 8(%esp), %xmm0; \
	    sub $8, %esp;         \
	    vmovq %xmm0, (%esp);
	#elif defined SSE2
	  #define REPUSH_ARGS \
	    movq 8(%esp), %xmm0; \
	    sub $8, %esp;        \
	    movq %xmm0, (%esp);
	#else
	  #define REPUSH_ARGS \
	    mov  8(%esp), %eax; /* arg1 */ \
	    mov 12(%esp), %ecx; /* arg2 */ \
	    sub $8, %esp;                  \
	    mov %ecx, 4(%esp);             \
	    mov %eax, (%esp);
	#endif
	
	#define OVERRIDE_CALLBACK_DEFINE(header, ary_idx, func) \
		OVERRIDE_FUNC_DEFINE(header, ary_idx)   \
		                                        \
		/* Get GOT base address. */ ;           \
		GET_GOT_ADDR                            \
		                                        \
		/* Get 2nd argument. */ ;               \
		mov 12(%esp), %eax;                     \
		push %eax                               \
		/* Call before callback function. */ ;  \
		DO_JMP_OVERRIDE(func)                   \
		/* Remove 1st argument from stack. */ ; \
		addl $0x4, %esp;                        \
		                                        \
		/* Get callback target. */ ;            \
		GET_NEXT_JMP_FUNC(header, ary_idx, 16)  \
		/* push original arguments to stack. */ \
		/* call override target. */ ;           \
		REPUSH_ARGS                             \
		call *%edx;                             \
		add $8, %esp;                           \
		                                        \
		/* Get after callback. */ ;             \
		GET_NEXT_JMP_FUNC(header, ary_idx, 20)  \
		                                        \
		/* null check for callback. */ ;        \
		jz .RETURN_CALLER_FUNC;                 \
		                                        \
		/* Call after callback. */              \
		call *%edx;                             \
		jmp .RETURN_CALLER_FUNC;
	
	/* Macro for hook each target function. */
	
	#define OVERRIDE_DO_BLK_CAREFUL(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForSweep, 4)
	#define OVERRIDE_OOP_FOLLOW_CONTENTS(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForParallel, 4)
	#define OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForParOld, 8)
	#define OVERRIDE_OOP_OOP_ITERATE(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForIterate, 4)
	#define OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(header, ary_idx, valid_caller) \
		OVERRIDE_DEFINE_WITH_CALLER_CHECK(header, ary_idx, callbackForIterate, 4, valid_caller)
	#define OVERRIDE_OOP_PAR_ADJUST(header, ary_idx) \
		OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForAdjustPtr, 4)
	#define OVERRIDE_OOP_DO_ADDR(header, ary_idx) \
		OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForDoAddr, 4)
	#define OVERRIDE_OOP_UPDATE_PTR(header, ary_idx) \
		OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForUpdatePtr, 8)
	
	#define OVERRIDE_JVMTI_ITERATE(header, ary_idx) \
		OVERRIDE_DEFINE(header, ary_idx, callbackForJvmtiIterate, 4)
	
#endif

/* Begin of .text section. */
	.text

.LORIGINAL_FUNC:
#if defined(__amd64__)
	pop %r11
	DO_LOAD_REG
	jmp *%r11
#elif defined(__i386__)
	addl $0x4, %esp
	pop %ecx
	pop %ebx
	jmp *%ecx
#endif


.RETURN_CALLER_FUNC:
#if defined(__amd64__)
	/* Return caller. Return address exists on top of stack. */
	/* We need to pop return address from stack. */
	pop %r11
	jmp *%r11
#elif defined(__i386__)
	/* restore %ebx & stack */
	pop %ebx;

	/* Return caller. */ ;
	pop %edx;
	jmp *%edx;
#endif


/* Override definition. */

/* For Parallel. */

/* instanceKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 0)
/* objArrayKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 1)
/* typeArrayKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 2)
/* instanceRefKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 3)


/* For Parallel (CR6964458). */

/* InstanceKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR6964458_par, 0)
/* objArrayKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR6964458_par, 1)
/* typeArrayKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR6964458_par, 2)
/* InstanceRefKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR6964458_par, 3)
/* InstanceClassLoaderKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR6964458_par, 4)


/* For Parallel (CR8000213). */

/* InstanceKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR8000213_par, 0)
/* ObjArrayKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR8000213_par, 1)
/* TypeArrayKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR8000213_par, 2)
/* InstanceRefKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR8000213_par, 3)
/* InstanceClassLoaderKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(CR8000213_par, 4)


/* For ParallelOld. */

/* instanceKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 0)
/* objArrayKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 1)
/* typeArrayKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 2)
/* instanceRefKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 3)


/* For ParallelOld (CR6964458). */

/* InstanceKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR6964458_parOld, 0)
/* objArrayKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR6964458_parOld, 1)
/* typeArrayKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR6964458_parOld, 2)
/* InstanceRefKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR6964458_parOld, 3)
/* InstanceClassLoaderKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR6964458_parOld, 4)


/* For ParallelOld (CR8000213). */

/* InstanceKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR8000213_parOld, 0)
/* ObjArrayKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR8000213_parOld, 1)
/* TypeArrayKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR8000213_parOld, 2)
/* InstanceRefKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR8000213_parOld, 3)
/* InstanceClassLoaderKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(CR8000213_parOld, 4)


/* For CMS. */
/*
 * We want to hook sweep phase
 * and around "ContiguousSpace::par_oop_iterate()".
 */

/* SweepClosure::do_blk_careful(HeapWord*) */
OVERRIDE_DO_BLK_CAREFUL(cms_sweep, 0)

/* instanceKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 0)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 1)
/* typeArrayKlass::oop_oop_iterate(oopDesc*, OopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 2)
/* instanceRefKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 3)

/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) for after CR6964458 */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_cms_new, 0)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) for after CR6964458 */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_cms_new, 1)
/* typeArrayKlass::oop_oop_iterate(oopDesc*, OopClosure*) for after CR6964458 */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_cms_new, 2)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) for after CR6964458 */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_cms_new, 3)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_cms_new, 4)

/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) for after CR8000213 */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_cms_new, 0)
/* ObjArrayKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) for after CR8000213 */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_cms_new, 1)
/* TypeArrayKlass::oop_oop_iterate(oopDesc*, OopClosure*) for after CR8000213 */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_cms_new, 2)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) for after CR8000213 */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_cms_new, 3)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_cms_new, 4)


/* For G1. */

/* initial-mark */
/* G1ParScanAndMarkExtRootClosure::do_oop(oopDesc**) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(oopDesc**) */
OVERRIDE_CALLBACK_DEFINE(g1, 0, callbackForDoOop);
/* G1ParScanAndMarkExtRootClosure::do_oop(unsigned int*) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(unsigned int*) */
OVERRIDE_CALLBACK_DEFINE(g1, 1, callbackForDoNarrowOop);

/* concurrent-root-region-scan */
/* instanceKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 2)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 3)
/* instanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 4)

/* concurrent-mark / remark */
/* instanceKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 5)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 6)
/* typeArrayKlass::oop_oop_iterate(oopDesc*, OopClosure*) */
OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(g1, 7, VTableForTypeArrayOopClosure)
/* instanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 8)

/* For G1 (CR6964458). */

/* initial-mark */
/* G1ParScanAndMarkExtRootClosure::do_oop(oopDesc**) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(oopDesc**) */
OVERRIDE_CALLBACK_DEFINE(CR6964458_g1, 0, callbackForDoOop);
/* G1ParScanAndMarkExtRootClosure::do_oop(unsigned int*) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(unsigned int*) */
OVERRIDE_CALLBACK_DEFINE(CR6964458_g1, 1, callbackForDoNarrowOop);

/* concurrent-root-region-scan */
/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_g1, 2)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_g1, 3)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_g1, 4)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_g1, 5)

/* concurrent-mark / remark */
/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_g1, 6)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_g1, 7)
/* typeArrayKlass::oop_oop_iterate(oopDesc*, ExtendedOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(CR6964458_g1, 8, VTableForTypeArrayOopClosure)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_g1, 9)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR6964458_g1, 10)

/* For G1 (CR8000213). */

/* initial-mark */
/* G1ParScanAndMarkExtRootClosure::do_oop(oopDesc**) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(oopDesc**) */
OVERRIDE_CALLBACK_DEFINE(CR8000213_g1, 0, callbackForDoOop);
/* G1ParScanAndMarkExtRootClosure::do_oop(unsigned int*) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(unsigned int*) */
OVERRIDE_CALLBACK_DEFINE(CR8000213_g1, 1, callbackForDoNarrowOop);

/* concurrent-root-region-scan */
/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_g1, 2)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_g1, 3)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_g1, 4)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_g1, 5)

/* concurrent-mark / remark */
/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_g1, 6)
/* ObjArrayKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_g1, 7)
/* TypeArrayKlass::oop_oop_iterate(oopDesc*, ExtendedOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(CR8000213_g1, 8, VTableForTypeArrayOopClosure)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_g1, 9)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8000213_g1, 10)

/* For G1 (CR8027746). */

/* initial-mark */
/* G1ParScanAndMarkExtRootClosure::do_oop(oopDesc**) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(oopDesc**) */
OVERRIDE_CALLBACK_DEFINE(CR8027746_g1, 0, callbackForDoOop);
/* G1ParScanAndMarkExtRootClosure::do_oop(unsigned int*) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(unsigned int*) */
OVERRIDE_CALLBACK_DEFINE(CR8027746_g1, 1, callbackForDoNarrowOop);

/* concurrent-root-region-scan */
/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8027746_g1, 2)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8027746_g1, 3)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8027746_g1, 4)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8027746_g1, 5)

/* concurrent-mark / remark */
/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8027746_g1, 6)
/* ObjArrayKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8027746_g1, 7)
/* TypeArrayKlass::oop_oop_iterate(oopDesc*, ExtendedOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(CR8027746_g1, 8, VTableForTypeArrayOopClosure)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8027746_g1, 9)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8027746_g1, 10)

/* For G1 (JDK-8049421). */

/* initial-mark */
/* G1ParScanAndMarkExtRootClosure::do_oop(oopDesc**) */
/* G1ParCopyClosure<(G1Barrier)0, G1MarkFromRoot>::do_oop(oopDesc**) */
OVERRIDE_CALLBACK_DEFINE(CR8049421_g1, 0, callbackForDoOop);
/* G1ParScanAndMarkExtRootClosure::do_oop(unsigned int*) */
/* G1ParCopyClosure<(G1Barrier)0, G1MarkFromRoot>::do_oop(unsigned int*) */
OVERRIDE_CALLBACK_DEFINE(CR8049421_g1, 1, callbackForDoNarrowOop);

/* concurrent-root-region-scan */
/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8049421_g1, 2)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8049421_g1, 3)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8049421_g1, 4)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8049421_g1, 5)

/* concurrent-mark / remark */
/* InstanceKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8049421_g1, 6)
/* ObjArrayKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8049421_g1, 7)
/* TypeArrayKlass::oop_oop_iterate(oopDesc*, ExtendedOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(CR8049421_g1, 8, VTableForTypeArrayOopClosure)
/* InstanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8049421_g1, 9)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(CR8049421_g1, 10)

/* G1 events */

/* CMCleanUp::do_void() */
OVERRIDE_CALLBACK_DEFINE(g1Event, 0, callbackForG1Cleanup);
/* G1CollectedHeap::gc_prologue() */
OVERRIDE_CALLBACK_DEFINE(g1Event, 1, callbackForG1Full);
/* G1CollectedHeap::gc_epilogue() */
OVERRIDE_CALLBACK_DEFINE(g1Event, 2, callbackForG1FullReturn);

/* for Klass relocation. */

/* instanceKlassKlass::oop_adjust_pointers(oopDesc*) */
OVERRIDE_OOP_PAR_ADJUST(adj, 0)
/* objArrayKlassKlass::oop_adjust_pointers(oopDesc*) */
OVERRIDE_OOP_PAR_ADJUST(adj, 1)
/* arrayKlassKlass::oop_adjust_pointers(oopDesc*) */
OVERRIDE_OOP_PAR_ADJUST(adj, 2)

/* for Klass relocation if use ParallelOldGC. */

/* MoveAndUpdateClosure::do_addr(HeapWord*, unsigned long) */
OVERRIDE_OOP_DO_ADDR(adj, 3)

/* UpdateOnlyClosure::do_addr(HeapWord*, size_t) */
OVERRIDE_OOP_DO_ADDR(adj, 4)

/* instanceKlassKlass::oop_update_pointers(ParCompactionManager*, oopDesc*) */
OVERRIDE_OOP_UPDATE_PTR(adj, 5)
/* objArrayKlassKlass::oop_update_pointers(ParCompactionManager*, oopDesc*) */
OVERRIDE_OOP_UPDATE_PTR(adj, 6)
/* arrayKlassKlass::oop_update_pointers(ParCompactionManager*, oopDesc*) */
OVERRIDE_OOP_UPDATE_PTR(adj, 7)

/* for JVMTI IterateOverHeap. */

/* IterateOverHeapObjectClosure::do_object(oop o) */
OVERRIDE_JVMTI_ITERATE(jvmti, 0)

/* For inner GC start. */

/* ParallelScavengeHeap::accumulate_statistics_all_tlabs(void) */
OVERRIDE_CALLBACK_DEFINE(innerStart, 0, callbackForInnerGCStart)
/* CollectedHeap::accumulate_statistics_all_tlabs(void) */
OVERRIDE_CALLBACK_DEFINE(innerStart, 1, callbackForInnerGCStart)
/* GenCollectedHeap::gc_prologue(bool) */
OVERRIDE_CALLBACK_DEFINE(innerStart, 2, callbackForInnerGCStart)

/* For WatcherThread */
/* WatcherThread::run() */
OVERRIDE_CALLBACK_DEFINE(watcherThread, 0, callbackForWatcherThreadRun)


#ifdef AVOID__i686
	/* Define again. */
	#undef AVOID__i686
	#define __i686
#endif