/*!
 * \file overrideFunc.S
 * \brief This file is used to override JVM inner function.<br>
 *        The function defined this file, used with v-table hook.<br>
 *        So in this file, all function is written by only assembler and macro.
 * Copyright (C) 2011-2015 Nippon Telegraph and Telephone Corporation
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 *
 */

/* Override functions of "do_oop" families. */
    .file "overrideFunc.S"

#define OVERRIDE_FUNC_DEFINE(header, ary_idx) \
        .global header##_override_func_##ary_idx ;         \
        .type header##_override_func_##ary_idx, @function; \
    header##_override_func_##ary_idx: ;                    \
    push %rbp;                                             \
    mov %rsp, %rbp;

#if defined(__amd64__)
    
    /* Macro for function parts define. */
    
    #define DO_SAVE_REG \
        sub $24, %rsp;                          \
        mov %rdx, 16(%rsp); /* 3rd argument. */ \
        mov %rsi,  8(%rsp); /* 2nd argument. */ \
        mov %rdi,   (%rsp); /* 1st argument. */
    
    #define DO_LOAD_REG \
        mov   (%rsp), %rdi; /* 1st argument. */ \
        mov  8(%rsp), %rsi; /* 2nd argument. */ \
        mov 16(%rsp), %rdx; /* 3rd argument. */ \
        add $24, %rsp;
    
    #define GET_HOOK_REC(header, ary_idx) \
        /* get and set override info */ ;                              \
        mov header##_hook@GOTPCREL(%rip), %r11;                        \
        mov (%r11), %r11;                                              \
        lea ary_idx*64(%r11), %r11; /* 64=sizeof(THookFunctionInfo) */
    
    #define GET_SOURCE_FUNC(header, ary_idx) \
        GET_HOOK_REC(header, ary_idx)                             ; \
        mov 32(%r11), %rax; /* Original vfunc offset in entry. */ ; \
        push %rax;                                                  
    
    #define GET_ALL_FUNC_ADDR(header, ary_idx) \
        GET_HOOK_REC(header, ary_idx)                                   ; \
        sub $16, %rsp;                                                    \
        movdqa 32(%r11), %xmm0; /* Copy Original&After func to XMM. */    \
        movdqu %xmm0, (%rsp);   /* Store Original&After func to stack. */ 
    
    #define DO_CHECK_PERM(oop_ofs) \
        mov is_in_permanent@GOTPCREL(%rip), %r11; \
        mov collectedHeap@GOTPCREL(%rip), %rdi;   \
        mov (%rdi), %rdi;                         \
        mov oop_ofs(%rsp), %rsi;                  \
        call *(%r11);                             \
        test %al, %al;                            
    
    #define DO_JMP_OVERRIDE(func, oop_ofs) \
        mov oop_ofs(%rsp), %rdi;         \
        mov func##@GOTPCREL(%rip), %r11; \
        call *%r11;                      
    
    #define DO_JMP_ANY_FUNC(nextFunc) \
        pop %r11;        \
        test %r11, %r11; \
        jz nextFunc;     \
        call *%r11;
    
    /* Macro for override function define. */
    
    #define OVERRIDE_DEFINE(header, ary_idx, func, oop_arg_idx) \
        OVERRIDE_FUNC_DEFINE(header, ary_idx) \
                                              \
        /* Save argument registers. */ ;      \
        DO_SAVE_REG                           \
                                              \
        /* Get and set override info. */ ;    \
        GET_SOURCE_FUNC(header, ary_idx)      \
                                              \
        /* Check permanent object. */ ;       \
        DO_CHECK_PERM(oop_arg_idx*8)          \
        jne .LORIGINAL_FUNC;                  \
                                              \
        /* Call callback function. */ ;       \
        DO_JMP_OVERRIDE(func, oop_arg_idx*8)  \
        jmp .LORIGINAL_FUNC                   

    #ifdef AVX
      #define CHECK_VTABLE(oop_reg, valid_caller) \
        vmovq (oop_reg), %xmm0;                  \
        vshufpd $0, %xmm0, %xmm1, %xmm1;         \
        mov valid_caller##@GOTPCREL(%rip), %r11; \
        vpcmpeqq (%r11), %xmm1, %xmm1;           \
        vptest %xmm1, %xmm1;
    #elif defined SSE4_1
      #define CHECK_VTABLE(oop_reg, valid_caller) \
        movq (oop_reg), %xmm0;                   \
        shufpd $0, %xmm0, %xmm1;                 \
        mov valid_caller##@GOTPCREL(%rip), %r11; \
        pcmpeqq (%r11), %xmm1;                   \
        ptest %xmm1, %xmm1;
    #else
      #define CHECK_VTABLE(oop_reg, valid_caller) \
        mov (oop_reg), oop_reg;                  \
        xor %rax, %rax;                          \
        mov valid_caller##@GOTPCREL(%rip), %r11; \
        cmp (%r11), oop_reg;                     \
        cmove %r11, %rax;                        \
        cmp 8(%r11), oop_reg;                    \
        cmove %r11, %rax;                        \
        test %rax, %rax;
    #endif

    #define OVERRIDE_DEFINE_WITH_CALLER_CHECK(header, ary_idx, func, oop_reg_idx, valid_caller) \
        OVERRIDE_FUNC_DEFINE(header, ary_idx)    \
                                                 \
        /* Save argument registers */            \
        DO_SAVE_REG                              \
                                                 \
        /* Get and set override info. */ ;       \
        GET_SOURCE_FUNC(header, ary_idx)         \
                                                 \
        /* Check vtable of caller */             \
        CHECK_VTABLE(%rdx, valid_caller);        \
        jz .LORIGINAL_FUNC;                      \
                                                 \
        /* Check permanent object. */ ;          \
        DO_CHECK_PERM(oop_reg_idx*8)             \
        jne .LORIGINAL_FUNC;                     \
                                                 \
        /* Call callback function. */ ;          \
        DO_JMP_OVERRIDE(func, oop_reg_idx*8)     \
        jmp .LORIGINAL_FUNC
    
    #define OVERRIDE_ADJ_DEFINE(header, ary_idx, func, oop_arg_idx) \
        OVERRIDE_FUNC_DEFINE(header, ary_idx) \
                                              \
        /* Save argument registers. */ ;      \
        DO_SAVE_REG                           \
                                              \
        /* Get and set override info. */ ;    \
        GET_SOURCE_FUNC(header, ary_idx)      \
                                              \
        /* Check permanent object. */ ;       \
        DO_CHECK_PERM(oop_arg_idx*8)          \
        jz .LORIGINAL_FUNC;                   \
                                              \
        /* Call callback function. */ ;       \
        DO_JMP_OVERRIDE(func, oop_arg_idx*8)  \
        jmp .LORIGINAL_FUNC                   
    
    #define OVERRIDE_CALLBACK_DEFINE(header, ary_idx, func, paramCnt) \
        OVERRIDE_FUNC_DEFINE(header, ary_idx)      \
        /* Stack alignment */                      \
        sub $8, %rsp;                              \
                                                   \
        /* Get all override info. */ ;             \
        GET_ALL_FUNC_ADDR(header, ary_idx)         \
                                                   \
        /* Save argument registers. */ ;           \
        DO_SAVE_REG                                \
                                                   \
        /* Call before callback with 2nd arg. */ ; \
        DO_JMP_OVERRIDE(func, 8)                   \
                                                   \
        /* Load argument registers. */ ;           \
        DO_LOAD_REG                                \
                                                   \
        /* Get callback target. */                 \
        pop %r11;                                  \
        /* call callback target. */                \
        callq *%r11;                               \
                                                   \
        /* After Callback */                       \
        DO_JMP_ANY_FUNC(.RETURN_CALLER_FUNC)       \
                                                   \
        /* Return to caller */                     \
        jmp .RETURN_CALLER_FUNC
    
    /* Macro for hook each target function. */
    
    #define OVERRIDE_DO_BLK_CAREFUL(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForSweep, 2)
    #define OVERRIDE_OOP_FOLLOW_CONTENTS(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForParallel, 2)
    #define OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForParOld, 3)
    #define OVERRIDE_OOP_OOP_ITERATE(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForIterate, 2)
    #define OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(header, ary_idx, valid_caller) \
        OVERRIDE_DEFINE_WITH_CALLER_CHECK(header, ary_idx, callbackForIterate, 2, valid_caller)
    
    #define OVERRIDE_OOP_PAR_ADJUST(header, ary_idx) \
        OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForAdjustPtr, 2)
    #define OVERRIDE_OOP_DO_ADDR(header, ary_idx) \
        OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForDoAddr, 2)
    #define OVERRIDE_OOP_UPDATE_PTR(header, ary_idx) \
        OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForUpdatePtr, 3)
    
    #define OVERRIDE_JVMTI_ITERATE(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForJvmtiIterate, 2)
    
#elif defined(__i386__)
    
    #ifdef __i686
        /*
         * This statement is for avoiding illegal conversion by preprocceor.
         * before : "call __i686.get_pc_thunk.bx;"
         * after  : "call 1.get_pc_thunk.bx;"
         */
        #undef __i686
        #define AVOID__i686
    #endif
    
    /* Macro for function parts define. */
    
    #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 6)
      #define GET_GOT_ADDR \
        push %ebx;                         \
        call __x86.get_pc_thunk.bx;        \
        addl $_GLOBAL_OFFSET_TABLE_, %ebx;
    #else
      #define GET_GOT_ADDR \
        push %ebx;                         \
        call __i686.get_pc_thunk.bx;       \
        addl $_GLOBAL_OFFSET_TABLE_, %ebx;
    #endif

    #define GET_HOOK_REC(header, ary_idx) \
        mov header##_hook@GOT(%ebx), %ecx;                             \
        mov (%ecx), %ecx;                                              \
        lea ary_idx*28(%ecx), %ecx; /* 28=sizeof(THookFunctionInfo) */ 
    
    #define GET_SOURCE_FUNC(header, ary_idx) \
        GET_HOOK_REC(header, ary_idx)                              \
        mov 16(%ecx), %eax; /* original hook offset in entry. */ ; \
        push %eax;                                                 
    
    #define DO_CHECK_PERM(oop_reg) \
        mov (oop_reg)+12(%esp), %edx;           \
        mov is_in_permanent@GOTOFF(%ebx), %ecx; \
        mov collectedHeap@GOTOFF(%ebx), %eax;   \
        push %edx;                              \
        push %eax; /* Set this pointer. */      \
        call *%ecx;                             \
        addl $0x04, %esp;                       \
        test %al, %al;                          
    
    #define DO_JMP_OVERRIDE(func) \
        mov func##@GOT(%ebx), %edx; \
        call *%edx;                 
    
    #define GET_NEXT_JMP_FUNC(header, ary_idx, offset) \
        GET_HOOK_REC(header, ary_idx) \
        mov offset(%ecx), %edx;       \
        test %edx, %edx;              \
    
    #define REENACT_ARGS(paramCnt) \
        /* Keep registers. */                \
        push %esi;                           \
        push %edi;                           \
        push %es;                            \
        push %ds;                            \
        pushf;                               \
        /* Set copy count. */                \
        mov $(paramCnt), %ecx;               \
        /* Set copy source position. */      \
        mov %esp, %eax;                      \
        addl $(paramCnt*4+20), %eax;         \
        mov %eax, %esi;                      \
        /* Set copy destination position. */ \
        mov %esp, %eax;                      \
        subl $0x04, %eax;                    \
        mov %eax, %edi;                      \
        /* Keep copy destination. */         \
        subl $(paramCnt*4), %esp;            \
        /* Copy arguments in stack. */       \
        std;                                 \
        rep movsd;                           
    
    #define RESTORE_ARGS(paramCnt) \
        /* Remove stack. */       \
        addl $(paramCnt*4), %esp; \
        /* Restore registers. */  \
        popf;                     \
        pop %ds;                  \
        pop %es;                  \
        pop %edi;                 \
        pop %esi;                 
    
    /* Macro for override function define. */
    
    #define OVERRIDE_DEFINE(header, ary_idx, func, oop_reg) \
        OVERRIDE_FUNC_DEFINE(header, ary_idx) \
                                              \
        /* Get GOT base address. */ ;         \
        GET_GOT_ADDR                          \
                                              \
        /* get and set override info. */ ;    \
        GET_SOURCE_FUNC(header, ary_idx)      \
                                              \
        /* Check permanent object. */ ;       \
        DO_CHECK_PERM(oop_reg)                \
        jne .LORIGINAL_FUNC;                  \
                                              \
        /* Call callback function. */ ;       \
        DO_JMP_OVERRIDE(func)                 \
        jmp .LORIGINAL_FUNC;

    #ifdef AVX
      #define CHECK_VTABLE(oop_reg, valid_caller) \
        vmovd (oop_reg), %xmm0;             \
        vpshufd $0, %xmm0, %xmm1;           \
        mov valid_caller##@GOT(%ebx), %ecx; \
        vmovq (%ecx), %xmm1;                \
        vpcmpeqd %xmm0, %xmm1, %xmm1;       \
        vptest %xmm1, %xmm1;
    #elif defined SSE4_1
      #define CHECK_VTABLE(oop_reg, valid_caller) \
        movd (oop_reg), %xmm0;              \
        pshufd $0, %xmm0, %xmm1;            \
        mov valid_caller##@GOT(%ebx), %ecx; \
        movq (%ecx), %xmm1;                 \
        pcmpeqd %xmm0, %xmm1;               \
        ptest %xmm1, %xmm1;
    #else
      #define CHECK_VTABLE(oop_reg, valid_caller) \
        mov (oop_reg), oop_reg;             \
        xor %edx, %edx;                     \
        mov valid_caller##@GOT(%ebx), %ecx; \
        cmp (%ecx), oop_reg;                \
        cmove %ecx, %edx;                   \
        cmp 4(%ecx), oop_reg;               \
        cmove %ecx, %edx;                   \
        test %edx, %edx;
    #endif

    #define OVERRIDE_DEFINE_WITH_CALLER_CHECK(header, ary_idx, func, oop_reg, valid_caller) \
        OVERRIDE_FUNC_DEFINE(header, ary_idx) \
                                              \
        /* Get GOT base address. */ ;         \
        GET_GOT_ADDR                          \
                                              \
        /* get and set override info */ ;     \
        GET_SOURCE_FUNC(header, ary_idx)      \
                                              \
        /* Get 3rd arg (OopClosure) */        \
        mov 20(%esp), %eax;                   \
                                              \
        /* Dummy stack for jmp */             \
        subl $4, %esp;                        \
        /* Check vtable of caller */          \
        CHECK_VTABLE(%eax, valid_caller);     \
        jz .LORIGINAL_FUNC;                   \
        addl $4, %esp;                        \
                                              \
        /* Check permanent object. */ ;       \
        DO_CHECK_PERM(oop_reg)                \
        jne .LORIGINAL_FUNC;                  \
                                              \
        /* Call callback function. */ ;       \
        DO_JMP_OVERRIDE(func)                 \
        jmp .LORIGINAL_FUNC;
    
    #define OVERRIDE_ADJ_DEFINE(header, ary_idx, func, oop_reg) \
        OVERRIDE_FUNC_DEFINE(header, ary_idx) \
                                              \
        /* Get GOT base address. */ ;         \
        GET_GOT_ADDR                          \
                                              \
        /* get and set override info. */ ;    \
        GET_SOURCE_FUNC(header, ary_idx)      \
                                              \
        /* Check permanent object. */ ;       \
        DO_CHECK_PERM(oop_reg)                \
        jz .LORIGINAL_FUNC;                   \
                                              \
        /* Call callback function. */ ;       \
        DO_JMP_OVERRIDE(func)                 \
        jmp .LORIGINAL_FUNC;

    /* repush arg1 & arg2 to stack */ \
    #ifdef AVX
      #define REPUSH_ARGS \
        vmovq 8(%esp), %xmm0; \
        sub $8, %esp;         \
        vmovq %xmm0, (%esp);
    #else
      #define REPUSH_ARGS \
        movq 8(%esp), %xmm0;  \
        sub $8, %esp;         \
        movq %xmm0, (%esp);
    #endif

    #define OVERRIDE_CALLBACK_DEFINE(header, ary_idx, func, paramCnt) \
      OVERRIDE_FUNC_DEFINE(header, ary_idx)   \
                                              \
      /* Get GOT base address. */ ;           \
      GET_GOT_ADDR                            \
                                              \
      /* Get 2nd argument. */ ;               \
      mov 12(%esp), %eax;                     \
      push %eax                               \
      /* Call before callback function. */ ;  \
      DO_JMP_OVERRIDE(func)                   \
      /* Remove 1st argument from stack. */ ; \
      addl $0x4, %esp;                        \
                                              \
      /* Get callback target. */ ;            \
      GET_NEXT_JMP_FUNC(header, ary_idx, 16)  \
      /* push original arguments to stack. */ \
      REPUSH_ARGS                             \
      /* call override target. */ ;           \
      call *%edx;                             \
      add $8, %esp;                           \
                                              \
      /* Get after callback. */ ;             \
      GET_NEXT_JMP_FUNC(header, ary_idx, 20)  \
                                              \
      /* null check for callback. */ ;        \
      jz .RETURN_CALLER_FUNC;                 \
                                              \
      /* Call after callback. */              \
      call *%edx;                             \
      jmp .RETURN_CALLER_FUNC;
 
    /* Macro for hook each target function. */
    
    #define OVERRIDE_DO_BLK_CAREFUL(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForSweep, 4)
    #define OVERRIDE_OOP_FOLLOW_CONTENTS(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForParallel, 4)
    #define OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForParOld, 8)
    #define OVERRIDE_OOP_OOP_ITERATE(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForIterate, 4)
    #define OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(header, ary_idx, valid_caller) \
        OVERRIDE_DEFINE_WITH_CALLER_CHECK(header, ary_idx, callbackForIterate, 4, valid_caller)
    
    #define OVERRIDE_OOP_PAR_ADJUST(header, ary_idx) \
        OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForAdjustPtr, 4)
    #define OVERRIDE_OOP_DO_ADDR(header, ary_idx) \
        OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForDoAddr, 4)
    #define OVERRIDE_OOP_UPDATE_PTR(header, ary_idx) \
        OVERRIDE_ADJ_DEFINE(header, ary_idx, callbackForUpdatePtr, 8)
    
    #define OVERRIDE_JVMTI_ITERATE(header, ary_idx) \
        OVERRIDE_DEFINE(header, ary_idx, callbackForJvmtiIterate, 4)
    
#endif

/* Begin of .text section. */
    .text

.LORIGINAL_FUNC:
#if defined(__amd64__)
    pop %r11;
    DO_LOAD_REG
    /* Restore stack pointer */
    mov %rbp, %rsp;
    /* Restore base pointer */
    pop %rbp;
    jmp *%r11;
#elif defined(__i386__)
    addl $0x4, %esp;
    pop %ecx;
    pop %ebx;
    jmp *%ecx;
#endif

.RETURN_CALLER_FUNC:;
#if defined(__amd64__)
    /* Restore stack pointer */
    mov %rbp, %rsp;
    /* Restore base pointer */
    pop %rbp;

    /*
     * Return caller. Return address exists on top of stack.
     * We need to pop return address from stack.
     */
    pop %r11;
    jmp *%r11;
#elif defined(__i386__)
    /* restore %edx & stack */
    pop %ebx;

    /* Return caller. */ ;
    pop %edx;
    jmp *%edx;
#endif

/* Override definition. */

/* For Parallel. */

/* instanceKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 0)
/* objArrayKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 1)
/* typeArrayKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 2)
/* instanceRefKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 3)
/* InstanceClassLoaderKlass::oop_follow_contents(oopDesc*) */
OVERRIDE_OOP_FOLLOW_CONTENTS(par, 4)

/* For ParallelOld. */

/* instanceKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 0)
/* objArrayKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 1)
/* typeArrayKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 2)
/* instanceRefKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 3)
/* InstanceClassLoaderKlass::oop_follow_contents(ParCompactionManager*, oopDesc*) */
OVERRIDE_PAR_OOP_FOLLOW_CONTENTS(parOld, 4)

/* For CMS. */
/*
 * We want to hook sweep phase
 * and around "ContiguousSpace::par_oop_iterate()".
 */

/* SweepClosure::do_blk_careful(HeapWord*) */
OVERRIDE_DO_BLK_CAREFUL(cms_sweep, 0)
/* instanceKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 0)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 1)
/* typeArrayKlass::oop_oop_iterate(oopDesc*, OopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 2)
/* instanceRefKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 3)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, Par_MarkRefsIntoAndScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(cms_new, 4)

/* For G1. */

/* initial-mark */
/* G1ParScanAndMarkExtRootClosure::do_oop(oopDesc**) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(oopDesc**) */
OVERRIDE_CALLBACK_DEFINE(g1, 0, callbackForDoOop, 2);
/* G1ParScanAndMarkExtRootClosure::do_oop(unsigned int*) */
/* G1ParCopyClosure<false, (G1Barrier)0, false>::do_oop(unsigned int*) */
OVERRIDE_CALLBACK_DEFINE(g1, 1, callbackForDoNarrowOop, 2);

/* concurrent-root-region-scan */
/* instanceKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 2)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 3)
/* instanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 4)

/* concurrent-mark / remark */
/* instanceKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 5)
/* objArrayKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 6)
/* typeArrayKlass::oop_oop_iterate(oopDesc*, OopClosure*) */
OVERRIDE_OOP_OOP_ITERATE_WITH_CALLER_CHECK(g1, 7, VTableForTypeArrayOopClosure)
/* instanceRefKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 8)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1RootRegionScanClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 9)
/* InstanceClassLoaderKlass::oop_oop_iterate_nv(oopDesc*, G1CMOopClosure*) */
OVERRIDE_OOP_OOP_ITERATE(g1, 10)

/* CMCleanUp::do_void() */
OVERRIDE_CALLBACK_DEFINE(g1Event, 0, callbackForG1Cleanup, 1);
/* G1CollectedHeap::gc_prologue() */
OVERRIDE_CALLBACK_DEFINE(g1Event, 1, callbackForG1Full, 2);
/* G1CollectedHeap::gc_epilogue() */
OVERRIDE_CALLBACK_DEFINE(g1Event, 2, callbackForG1FullReturn, 2);

/* for Klass relocation. */

/* instanceKlassKlass::oop_adjust_pointers(oopDesc*) */
OVERRIDE_OOP_PAR_ADJUST(adj, 0)
/* objArrayKlassKlass::oop_adjust_pointers(oopDesc*) */
OVERRIDE_OOP_PAR_ADJUST(adj, 1)
/* arrayKlassKlass::oop_adjust_pointers(oopDesc*) */
OVERRIDE_OOP_PAR_ADJUST(adj, 2)

/* for Klass relocation if use ParallelOldGC. */

/* MoveAndUpdateClosure::do_addr(HeapWord*, unsigned long) */
OVERRIDE_OOP_DO_ADDR(adj, 3)
/* UpdateOnlyClosure::do_addr(HeapWord*, size_t) */
OVERRIDE_OOP_DO_ADDR(adj, 4)

/* instanceKlassKlass::oop_update_pointers(ParCompactionManager*, oopDesc*) */
OVERRIDE_OOP_UPDATE_PTR(adj, 5)
/* objArrayKlassKlass::oop_update_pointers(ParCompactionManager*, oopDesc*) */
OVERRIDE_OOP_UPDATE_PTR(adj, 6)
/* arrayKlassKlass::oop_update_pointers(ParCompactionManager*, oopDesc*) */
OVERRIDE_OOP_UPDATE_PTR(adj, 7)

/* for JVMTI IterateOverHeap. */

/* IterateOverHeapObjectClosure::do_object(oop o) */
OVERRIDE_JVMTI_ITERATE(jvmti, 0)

/* For inner GC start. */

/* ParallelScavengeHeap::accumulate_statistics_all_tlabs(void) */
OVERRIDE_CALLBACK_DEFINE(innerStart, 0, callbackForInnerGCStart, 1)
/* CollectedHeap::accumulate_statistics_all_tlabs(void) */
OVERRIDE_CALLBACK_DEFINE(innerStart, 1, callbackForInnerGCStart, 1)
/* GenCollectedHeap::gc_prologue(bool) */
OVERRIDE_CALLBACK_DEFINE(innerStart, 2, callbackForInnerGCStart, 2)

/* For WatcherThread */
/* WatcherThread::run() */
OVERRIDE_CALLBACK_DEFINE(watcherThread, 0, callbackForWatcherThreadRun, 1)


#ifdef AVOID__i686
    
    /* Define again. */
    #undef AVOID__i686
    #define __i686
#endif