/*
 * Copyright (C) 2010 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include "ThunkGenerators.h"

#include "CodeBlock.h"
#include "JITOperations.h"
#include "JSArray.h"
#include "JSArrayIterator.h"
#include "JSStack.h"
#include "MathCommon.h"
#include "Operations.h"
#include "SpecializedThunkJIT.h"
#include <wtf/InlineASM.h>
#include <wtf/StringPrintStream.h>
#include <wtf/text/StringImpl.h>

#if ENABLE(JIT)

namespace JSC {

inline void emitPointerValidation(CCallHelpers& jit, GPRReg pointerGPR)
{
#if !ASSERT_DISABLED
    CCallHelpers::Jump isNonZero = jit.branchTestPtr(CCallHelpers::NonZero, pointerGPR);
    jit.breakpoint();
    isNonZero.link(&jit);
    jit.push(pointerGPR);
    jit.load8(pointerGPR, pointerGPR);
    jit.pop(pointerGPR);
#else
    UNUSED_PARAM(jit);
    UNUSED_PARAM(pointerGPR);
#endif
}

static void returnFromJavaScript(CCallHelpers& jit)
{
#if CPU(X86_64)
#if OS(WINDOWS)
    // JIT operations can use up to 6 args (4 in registers and 2 on the stack).
    // In addition, X86_64 ABI specifies that the worse case stack alignment
    // requirement is 32 bytes. Based on these factors, we need to pad the stack
    // and additional 40 bytes.
#   define EXTRA_STACK_SIZE              40
    jit.addPtr(CCallHelpers::TrustedImm32(8), X86Registers::esp);
#else
    // We don't need extra stack space for out-going args, but we need to align the
    // stack to 32 byte alignment, therefore add 8 bytes to the stack.
#   define EXTRA_STACK_SIZE              8
    jit.addPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), X86Registers::esp);
#endif

    jit.pop(X86Registers::ebx);
    jit.pop(X86Registers::r15);
    jit.pop(X86Registers::r14);
    jit.pop(X86Registers::r13);
    jit.pop(X86Registers::r12);
    jit.pop(X86Registers::ebp);
#elif CPU(X86)
    // JIT Operation can use up to 6 arguments right now. So, we need to
    // reserve space in this stack frame for the out-going args. To ensure that
    // the stack remains aligned on an 16 byte boundary, we round the padding up
    // by 28 bytes.
#   define EXTRA_STACK_SIZE 28

    jit.addPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), X86Registers::esp);

    jit.pop(X86Registers::ebx);
    jit.pop(X86Registers::edi);
    jit.pop(X86Registers::esi);
    jit.pop(X86Registers::ebp);
#elif CPU(ARM_TRADITIONAL)
    // JIT Operation can use up to 6 arguments right now. So, we need to
    // reserve space in this stack frame for the out-going args. We need to
    // add space for 16 more bytes.
#   define EXTRA_STACK_SIZE              16

    jit.addPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), ARMRegisters::sp);

    jit.pop(ARMRegisters::lr);
    jit.pop(ARMRegisters::r11);
    jit.pop(ARMRegisters::r10);
    jit.pop(ARMRegisters::r9);
    jit.pop(ARMRegisters::r8);
    jit.pop(ARMRegisters::r6);
    jit.pop(ARMRegisters::r5);
    jit.pop(ARMRegisters::r4);
#elif CPU(ARM_THUMB2)
    // JIT Operation can use up to 6 arguments right now. So, we need to
    // reserve space in this stack frame for the out-going args. To ensure that
    // the stack remains aligned on an 16 byte boundary, we round the padding up
    // by 28 bytes.
#   define EXTRA_STACK_SIZE              28

    jit.addPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), ARMRegisters::sp);
    jit.pop(ARMRegisters::r11);
    jit.pop(ARMRegisters::r10);
    jit.pop(ARMRegisters::r9);
    jit.pop(ARMRegisters::r8);
    jit.pop(ARMRegisters::r7);
    jit.pop(ARMRegisters::r6);
    jit.pop(ARMRegisters::r5);
    jit.pop(ARMRegisters::r4);
    jit.pop(ARMRegisters::lr);
#elif CPU(ARM64)
    jit.pop(ARM64Registers::x28);
    jit.pop(ARM64Registers::x27);
    jit.pop(ARM64Registers::x26);
    jit.pop(ARM64Registers::x25);
    jit.pop(ARM64Registers::x24);
    jit.pop(ARM64Registers::x23);
    jit.pop(ARM64Registers::x22);
    jit.pop(ARM64Registers::x21);
    jit.pop(ARM64Registers::x20);
    jit.pop(ARM64Registers::x19);
    jit.pop(ARM64Registers::lr);
#elif CPU(MIPS)
#   define PRESERVED_GP_OFFSET         60
#   define PRESERVED_S0_OFFSET         64
#   define PRESERVED_S1_OFFSET         68
#   define PRESERVED_S2_OFFSET         72
#   define PRESERVED_S3_OFFSET         76
#   define PRESERVED_S4_OFFSET         80
#   define PRESERVED_RETURN_ADDRESS_OFFSET 84
#   define PRESERVED_FP_OFFSET         88
#   define STACK_LENGTH               112

    jit.loadPtr(CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S0_OFFSET), MIPSRegisters::s0);
    jit.loadPtr(CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S1_OFFSET), MIPSRegisters::s1);
    jit.loadPtr(CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S2_OFFSET), MIPSRegisters::s2);
    jit.loadPtr(CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S3_OFFSET), MIPSRegisters::s3);
    jit.loadPtr(CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S4_OFFSET), MIPSRegisters::s4);
    jit.loadPtr(CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_RETURN_ADDRESS_OFFSET), MIPSRegisters::ra);
    jit.loadPtr(CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_FP_OFFSET), MIPSRegisters::fp);
	jit.addPtr(CCallHelpers::TrustedImm32(STACK_LENGTH), MIPSRegisters::sp);
#elif CPU(SH4)
#   define EXTRA_STACK_SIZE 20

    jit.addPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), SH4Registers::sp);
    jit.pop(SH4Registers::r8);
    jit.pop(SH4Registers::r9);
    jit.pop(SH4Registers::r10);
    jit.pop(SH4Registers::r11);
    jit.pop(SH4Registers::r13);
    jit.pop(SH4Registers::pr);
    jit.pop(SH4Registers::fp);
#endif

    jit.ret();
}

MacroAssemblerCodeRef callToJavaScript(VM* vm)
{
    // The signature of the code generated is EncodedJSValue callToJavaScript(void* code, CallFrame*)

    CCallHelpers jit(vm);

#if CPU(X86_64)
#if OS(WINDOWS)
    // Dump register parameters to their home address
    jit.storePtr(X86Registers::r9, CCallHelpers::Address(X86Registers::esp, 0x20));
    jit.storePtr(X86Registers::r8, CCallHelpers::Address(X86Registers::esp, 0x18));
    jit.storePtr(X86Registers::edx, CCallHelpers::Address(X86Registers::esp, 0x10));
    jit.storePtr(X86Registers::ecx, CCallHelpers::Address(X86Registers::esp, 0x8));
#endif // OS(WINDOWS)
    jit.push(X86Registers::ebp);
    jit.move(X86Registers::ebp, GPRInfo::nonArgGPR0);
    jit.move(X86Registers::esp, X86Registers::ebp);
    jit.push(X86Registers::r12);
    jit.push(X86Registers::r13);
    jit.push(X86Registers::r14);
    jit.push(X86Registers::r15);
    jit.push(X86Registers::ebx);

    jit.subPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), X86Registers::esp);

#   define CALLFRAME_SRC_REG GPRInfo::argumentGPR1
#   define PREVIOUS_CALLFRAME_REG GPRInfo::nonArgGPR0
#elif CPU(X86)
    jit.push(X86Registers::ebp);
    jit.move(X86Registers::ebp, GPRInfo::nonArgGPR0);
    jit.move(X86Registers::esp, X86Registers::ebp);
    jit.push(X86Registers::esi);
    jit.push(X86Registers::edi);
    jit.push(X86Registers::ebx);

    jit.subPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), X86Registers::esp);

    jit.load32(CCallHelpers::Address(X86Registers::esp, EXTRA_STACK_SIZE + 24), GPRInfo::callFrameRegister);

#   define CALLFRAME_SRC_REG GPRInfo::callFrameRegister
#   define CALL_CODE jit.call(CCallHelpers::Address(X86Registers::esp, EXTRA_STACK_SIZE + 20))
#   define PREVIOUS_CALLFRAME_REG GPRInfo::nonArgGPR0
#elif CPU(ARM_TRADITIONAL)
    jit.push(ARMRegisters::r4);
    jit.push(ARMRegisters::r5);
    jit.push(ARMRegisters::r6);
    jit.push(ARMRegisters::r8);
    jit.push(ARMRegisters::r9);
    jit.push(ARMRegisters::r10);
    jit.push(ARMRegisters::r11);
    jit.push(ARMRegisters::lr);
    jit.move(ARMRegisters::r11, GPRInfo::nonArgGPR0);
    jit.subPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), ARMRegisters::sp);

#   define CALLFRAME_SRC_REG GPRInfo::argumentGPR1
#   define PREVIOUS_CALLFRAME_REG GPRInfo::nonArgGPR0
#elif CPU(ARM_THUMB2)
    jit.push(ARMRegisters::lr);
    jit.push(ARMRegisters::r4);
    jit.push(ARMRegisters::r5);
    jit.push(ARMRegisters::r6);
    jit.push(ARMRegisters::r7);
    jit.push(ARMRegisters::r8);
    jit.push(ARMRegisters::r9);
    jit.push(ARMRegisters::r10);
    jit.push(ARMRegisters::r11);
    jit.move(ARMRegisters::r7, GPRInfo::nonArgGPR0);
    jit.subPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), ARMRegisters::sp);

#   define CALLFRAME_SRC_REG GPRInfo::argumentGPR1
#   define PREVIOUS_CALLFRAME_REG GPRInfo::nonArgGPR0
#elif CPU(ARM64)
    jit.push(ARM64Registers::lr);
    jit.push(ARM64Registers::x19);
    jit.push(ARM64Registers::x20);
    jit.push(ARM64Registers::x21);
    jit.push(ARM64Registers::x22);
    jit.push(ARM64Registers::x23);
    jit.push(ARM64Registers::x24);
    jit.push(ARM64Registers::x25);
    jit.push(ARM64Registers::x26);
    jit.push(ARM64Registers::x27);
    jit.push(ARM64Registers::x28);
    jit.push(ARM64Registers::x29);
    jit.move(ARM64Registers::x29, GPRInfo::nonArgGPR0);

#   define CALLFRAME_SRC_REG GPRInfo::argumentGPR1
#   define PREVIOUS_CALLFRAME_REG GPRInfo::nonArgGPR0
#elif CPU(MIPS)
    jit.subPtr(CCallHelpers::TrustedImm32(STACK_LENGTH), MIPSRegisters::sp);
    jit.storePtr(MIPSRegisters::fp, CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_FP_OFFSET));
    jit.storePtr(MIPSRegisters::ra, CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_RETURN_ADDRESS_OFFSET));
    jit.storePtr(MIPSRegisters::s4, CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S4_OFFSET));
    jit.storePtr(MIPSRegisters::s3, CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S3_OFFSET));
    jit.storePtr(MIPSRegisters::s2, CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S2_OFFSET));
    jit.storePtr(MIPSRegisters::s1, CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S1_OFFSET));
    jit.storePtr(MIPSRegisters::s0, CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_S0_OFFSET));
#if WTF_MIPS_PIC
    jit.storePtr(MIPSRegisters::gp, CCallHelpers::Address(MIPSRegisters::sp, PRESERVED_GP_OFFSET));
#endif
    jit.move(MIPSRegisters::fp, GPRInfo::nonArgGPR0);

#   define CALLFRAME_SRC_REG GPRInfo::argumentGPR1
#   define PREVIOUS_CALLFRAME_REG GPRInfo::nonArgGPR0
#elif CPU(SH4)
    jit.push(SH4Registers::fp);
    jit.push(SH4Registers::pr);
    jit.push(SH4Registers::r13);
    jit.push(SH4Registers::r11);
    jit.push(SH4Registers::r10);
    jit.push(SH4Registers::r9);
    jit.push(SH4Registers::r8);
    jit.subPtr(CCallHelpers::TrustedImm32(EXTRA_STACK_SIZE), SH4Registers::sp);
    jit.move(SH4Registers::fp, GPRInfo::nonArgGPR0);

#   define CALLFRAME_SRC_REG GPRInfo::argumentGPR1
#   define PREVIOUS_CALLFRAME_REG GPRInfo::nonArgGPR0
#endif

    if (CALLFRAME_SRC_REG != GPRInfo::callFrameRegister)
        jit.move(CALLFRAME_SRC_REG, GPRInfo::callFrameRegister);
    jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister), GPRInfo::nonArgGPR1);
    jit.storePtr(PREVIOUS_CALLFRAME_REG, CCallHelpers::Address(GPRInfo::nonArgGPR1));

#if USE(JSVALUE64)
    jit.move(CCallHelpers::TrustedImm64(0xffff000000000000L), GPRInfo::tagTypeNumberRegister);
    jit.move(CCallHelpers::TrustedImm64(0xffff000000000002L), GPRInfo::tagMaskRegister);
#endif

#if defined(CALL_CODE)
    CALL_CODE;
#else
    jit.call(GPRInfo::argumentGPR0);
#endif

#undef CALLFRAME_SRC_REG
#undef CALL_CODE
#undef PREVIOUS_CALLFRAME_REG

    returnFromJavaScript(jit);

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(patchBuffer, ("Call to JavaScript thunk"));
}

MacroAssemblerCodeRef throwNotCaught(VM* vm)
{
    CCallHelpers jit(vm);

    returnFromJavaScript(jit);

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(patchBuffer, ("Uncaught throw thunk"));
}

// We will jump here if the JIT code tries to make a call, but the
// linking helper (C++ code) decides to throw an exception instead.
MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(VM* vm)
{
    CCallHelpers jit(vm);

    // The call pushed a return address, so we need to pop it back off to re-align the stack,
    // even though we won't use it.
    jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);

    // The CallFrame register points to the (failed) callee frame, so we need to pop back one frame.
    jit.emitGetCallerFrameFromCallFrameHeaderPtr(GPRInfo::callFrameRegister);

    jit.setupArguments(CCallHelpers::TrustedImmPtr(vm), GPRInfo::callFrameRegister);
    jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0);
    emitPointerValidation(jit, GPRInfo::nonArgGPR0);
    jit.call(GPRInfo::nonArgGPR0);
    jit.jumpToExceptionHandler();

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(patchBuffer, ("Throw exception from call slow path thunk"));
}

static void slowPathFor(
    CCallHelpers& jit, VM* vm, P_JITOperation_ECli slowPathFunction)
{
    jit.preserveReturnAddressAfterCall(GPRInfo::nonArgGPR2);
    emitPointerValidation(jit, GPRInfo::nonArgGPR2);
    jit.emitPutReturnPCToCallFrameHeader(GPRInfo::nonArgGPR2);
    jit.storePtr(GPRInfo::callFrameRegister, &vm->topCallFrame);
    jit.setupArgumentsWithExecState(GPRInfo::regT2);
    jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0);
    emitPointerValidation(jit, GPRInfo::nonArgGPR0);
    jit.call(GPRInfo::nonArgGPR0);

    // This slow call will return the address of one of the following:
    // 1) Exception throwing thunk.
    // 2) Host call return value returner thingy.
    // 3) The function to call.
    jit.emitGetReturnPCFromCallFrameHeaderPtr(GPRInfo::nonPreservedNonReturnGPR);
	jit.emitPutReturnPCToCallFrameHeader(CCallHelpers::TrustedImmPtr(0));
    emitPointerValidation(jit, GPRInfo::nonPreservedNonReturnGPR);
    jit.restoreReturnAddressBeforeReturn(GPRInfo::nonPreservedNonReturnGPR);
    emitPointerValidation(jit, GPRInfo::returnValueGPR);
    jit.jump(GPRInfo::returnValueGPR);
}

MacroAssemblerCodeRef linkCallThunkGenerator(VM* vm)
{
    // The return address is on the stack or in the link register. We will hence
    // save the return address to the call frame while we make a C++ function call
    // to perform linking and lazy compilation if necessary. We expect the callee
	// to be in regT0/regT1 (payload/tag), the CallFrame to have already
	// been adjusted, and all other registers to be available for use.

    CCallHelpers jit(vm);

    slowPathFor(jit, vm, operationLinkCall);

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(patchBuffer, ("Link call slow path thunk"));
}

// For closure optimizations, we only include calls, since if you're using closures for
// object construction then you're going to lose big time anyway.
MacroAssemblerCodeRef linkPolymorphicCallThunkGenerator(VM* vm)
{
    CCallHelpers jit(vm);

    slowPathFor(jit, vm, operationLinkPolymorphicCall);

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(patchBuffer, ("Link polymorphic call slow path thunk"));
}

MacroAssemblerCodeRef virtualThunkFor(VM* vm, CallLinkInfo& callLinkInfo)
{
    // The callee is in regT0 (for JSVALUE32_64, the tag is in regT1).
    // The return address is on the stack, or in the link register. We will hence
    // jump to the callee, or save the return address to the call frame while we
    // make a C++ function call to the appropriate JIT operation.

    CCallHelpers jit(vm);

    CCallHelpers::JumpList slowCase;

    // This is a slow path execution, and regT2 contains the CallLinkInfo. Count the
    // slow path execution for the profiler.
    jit.add32(
        CCallHelpers::TrustedImm32(1),
        CCallHelpers::Address(GPRInfo::regT2, CallLinkInfo::offsetOfSlowPathCount()));

    // FIXME: we should have a story for eliminating these checks. In many cases,
    // the DFG knows that the value is definitely a cell, or definitely a function.

#if USE(JSVALUE64)
    jit.move(CCallHelpers::TrustedImm64(TagMask), GPRInfo::regT4);

    slowCase.append(
        jit.branchTest64(
            CCallHelpers::NonZero, GPRInfo::regT0, GPRInfo::regT4));
#else
    slowCase.append(
        jit.branch32(
            CCallHelpers::NotEqual, GPRInfo::regT1,
            CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
    jit.loadPtr(CCallHelpers::Address(GPRInfo::regT0, JSCell::structureOffset()), GPRInfo::regT4);
    slowCase.append(
		jit.branch8(
			CCallHelpers::NotEqual,
			CCallHelpers::Address(GPRInfo::regT4, Structure::typeInfoTypeOffset()),
			CCallHelpers::TrustedImm32(JSFunctionType)));

    // Now we know we have a JSFunction.

    jit.loadPtr(
        CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()),
        GPRInfo::regT4);
    jit.loadPtr(
		CCallHelpers::Address(
			GPRInfo::regT4, ExecutableBase::offsetOfJITCodeWithArityCheckFor(
				callLinkInfo.specializationKind(), callLinkInfo.registerPreservationMode())),
		GPRInfo::regT4);
	slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT4));

    // Now we know that we have a CodeBlock, and we're committed to making a fast
    // call.

#if USE(JSVALUE64)
    jit.store64(
		JSInterfaceJIT::TrustedImm64(ValueUndefined),
        CCallHelpers::Address(
            GPRInfo::callFrameRegister,
            static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain));
#else
    jit.storePtr(
		AssemblyHelpers::TrustedImmPtr(0),
        CCallHelpers::Address(
            GPRInfo::callFrameRegister,
            static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain +
            OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
    jit.store32(
        CCallHelpers::TrustedImm32(JSValue::UndefinedTag),
        CCallHelpers::Address(
            GPRInfo::callFrameRegister,
            static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain +
            OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
#endif

    // Make a tail call. This will return back to JIT code.
    emitPointerValidation(jit, GPRInfo::regT4);
    jit.jump(GPRInfo::regT4);

    slowCase.link(&jit);

    // Here we don't know anything, so revert to the full slow path.

    slowPathFor(jit, vm, operationVirtualCall);

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(
		patchBuffer,
        ("Virtual %s%s slow path thunk at CodePtr(%p)",
        callLinkInfo.specializationKind() == CodeForCall ? "call" : "construct",
        callLinkInfo.registerPreservationMode() == MustPreserveRegisters ? " that preserves registers" : "",
        callLinkInfo.callReturnLocation().dataLocation()));
}

static MacroAssemblerCodeRef nativeForGenerator(JSGlobalData* vm, CodeSpecializationKind kind)
{
    int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);

    JSInterfaceJIT jit(vm);

    jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock);
    jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);

#if CPU(X86)
    jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0);
    jit.emitPutUndefinedToCallFrameHeader(JSStack::ScopeChain);

    jit.peek(JSInterfaceJIT::regT1);
    jit.emitPutReturnPCToCallFrameHeader(JSInterfaceJIT::regT1);

    // Calling convention:      f(ecx, edx, ...);
    // Host function signature: f(ExecState*);
    jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);

    jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister); // Align stack after call.

    // call the function
    jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
    jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
    jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
    jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));

    jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister);

#elif CPU(X86_64)
    jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0);
    jit.emitPutUndefinedToCallFrameHeader(JSStack::ScopeChain);

    jit.peek(JSInterfaceJIT::regT1);
    jit.emitPutReturnPCToCallFrameHeader(JSInterfaceJIT::regT1);

    // Calling convention:      f(edi, esi, edx, ecx, ...);
    // Host function signature: f(ExecState*);
    jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);

    jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); // Align stack after call.

    jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
    jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
    jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
    jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));

    jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);

#elif CPU(ARM64)
    COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT1, prev_callframe_not_trampled_by_T1);
    COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT3, prev_callframe_not_trampled_by_T3);
    COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
    COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
    COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);

    // Load caller frame's scope chain into this callframe so that whatever we call can
    // get to its global data.
    jit.emitGetCallerFrameFromCallFrameHeaderPtr(ARM64Registers::x3);
    jit.emitPutUndefinedToCallFrameHeader(JSStack::ScopeChain);

    jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved
    jit.emitPutReturnPCToCallFrameHeader(ARM64Registers::lr);

    // Host function signature: f(ExecState*);
    jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);

    jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1);
    jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
    jit.move(ARM64Registers::x3, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
    jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));

    jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);

#elif CPU(ARM) || CPU(SH4) || CPU(MIPS)
    // Load caller frame's scope chain into this callframe so that whatever we call can get to its global data.
    jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT2);
    jit.emitPutUndefinedToCallFrameHeader(JSStack::ScopeChain);

    jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved
    jit.emitPutReturnPCToCallFrameHeader(JSInterfaceJIT::regT3);

#if CPU(MIPS)
    // Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
    jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif

    // Calling convention is f(argumentGPR0, argumentGPR1, ...).
    // Host function signature is f(ExecState*).
    jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);

    jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1);
    jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
    jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
    jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));

#if CPU(MIPS)
    // Restore stack space
    jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif

    jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
#else
#error "JIT not supported on this platform."
    UNUSED_PARAM(executableOffsetToFunction);
    breakpoint();
#endif

    // Check for an exception
#if USE(JSVALUE64)
    jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
    JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
    JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
        JSInterfaceJIT::NotEqual,
        JSInterfaceJIT::AbsoluteAddress(vm->addressOfException()),
        JSInterfaceJIT::TrustedImm32(0));
#endif

    // Return.
    jit.ret();

    // Handle an exception
    exceptionHandler.link(&jit);

    // Grab the return address.
    jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT1);

    jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);

#if CPU(X86) && USE(JSVALUE32_64)
    jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
    jit.push(JSInterfaceJIT::callFrameRegister);
#else
    jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
#endif
    jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
    jit.call(JSInterfaceJIT::regT3);
#if CPU(X86) && USE(JSVALUE32_64)
    jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif

    jit.jumpToExceptionHandler();

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(patchBuffer, ("native %s trampoline", toCString(kind).data()));
}

MacroAssemblerCodeRef nativeCallGenerator(JSGlobalData* vm)
{
    return nativeForGenerator(vm, CodeForCall);
}

MacroAssemblerCodeRef nativeConstructGenerator(JSGlobalData* vm)
{
    return nativeForGenerator(vm, CodeForConstruct);
}

MacroAssemblerCodeRef arityFixupGenerator(JSGlobalData* vm)
{
    JSInterfaceJIT jit(vm);

    // We enter with fixup count in regT0
#if USE(JSVALUE64)
#  if CPU(X86_64)
    jit.pop(JSInterfaceJIT::regT4);
#  endif
    jit.neg64(JSInterfaceJIT::regT0);
	jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
	jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2);
    jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);

    // Adjust call frame register and stack pointer to account for missing args.
    // We need to change the stack pointer first before performing copy/fill loops.
    // This stack space below the stack pointer is considered unused by OS. Therefore,
    // OS may corrupt this space when constructing a signal stack.
    jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1);
    jit.lshift64(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT1);
    jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::callFrameRegister);

    // Move current frame down regT0 number of slots
    JSInterfaceJIT::Label copyLoop(jit.label());
    jit.load64(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
    jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
    jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
    jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);

    // Fill in regT0 missing arg slots with undefined
    jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
    jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT1);
    JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
    jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
    jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
    jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);

#  if CPU(X86_64)
    jit.push(JSInterfaceJIT::regT4);
#  endif
    jit.ret();
#else
#  if CPU(X86)
    jit.pop(JSInterfaceJIT::regT4);
#  endif
    jit.neg32(JSInterfaceJIT::regT0);
	jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
	jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2);
    jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);

    // Adjust call frame register and stack pointer to account for missing args.
    // We need to change the stack pointer first before performing copy/fill loops.
    // This stack space below the stack pointer is considered unused by OS. Therefore,
    // OS may corrupt this space when constructing a signal stack.
    jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1);
    jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT1);
    jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::callFrameRegister);

    // Move current frame down regT0 number of slots
    JSInterfaceJIT::Label copyLoop(jit.label());
    jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
    jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
    jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT1);
    jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
    jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
    jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);

    // Fill in regT0 missing arg slots with undefined
    jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
    JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
    jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT1);
    jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
    jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1);
    jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));

    jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
    jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);

#  if CPU(X86)
    jit.push(JSInterfaceJIT::regT4);
#  endif
    jit.ret();
#endif

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}

MacroAssemblerCodeRef stringGetByValGenerator(VM* vm)
{
    // regT0 is JSString*, and regT1 (64bit) or regT2 (32bit) is int index.
    // Return regT0 = result JSString* if succeeds. Otherwise, return regT0 = 0.
#if USE(JSVALUE64)
    GPRReg stringGPR = GPRInfo::regT0;
    GPRReg indexGPR = GPRInfo::regT1;
    GPRReg scratchGPR = GPRInfo::regT2;
#else
    GPRReg stringGPR = GPRInfo::regT0;
    GPRReg indexGPR = GPRInfo::regT2;
    GPRReg scratchGPR = GPRInfo::regT1;
#endif

    JSInterfaceJIT jit(vm);
    JSInterfaceJIT::JumpList failures;
    //jit.tagReturnAddress();

    // Load string length to regT2, and start the process of loading the data pointer into regT0
    jit.load32(JSInterfaceJIT::Address(stringGPR, JSString::offsetOfLength()), scratchGPR);
    jit.loadPtr(JSInterfaceJIT::Address(stringGPR, JSString::offsetOfValue()), stringGPR);
    failures.append(jit.branchTestPtr(JSInterfaceJIT::Zero, stringGPR));

    // Do an unsigned compare to simultaneously filter negative indices as well as indices that are too large
    failures.append(jit.branch32(JSInterfaceJIT::AboveOrEqual, indexGPR, scratchGPR));

    // Load the character
    JSInterfaceJIT::JumpList is16Bit;
    JSInterfaceJIT::JumpList cont8Bit;
    // Load the string flags
    jit.load32(JSInterfaceJIT::Address(stringGPR, StringImpl::flagsOffset()), scratchGPR);
    jit.loadPtr(JSInterfaceJIT::Address(stringGPR, StringImpl::dataOffset()), stringGPR);
    is16Bit.append(jit.branchTest32(JSInterfaceJIT::Zero, scratchGPR, JSInterfaceJIT::TrustedImm32(StringImpl::flagIs8Bit())));
    jit.load8(JSInterfaceJIT::BaseIndex(stringGPR, indexGPR, JSInterfaceJIT::TimesOne, 0), stringGPR);
    cont8Bit.append(jit.jump());
    is16Bit.link(&jit);
    jit.load16(JSInterfaceJIT::BaseIndex(stringGPR, indexGPR, JSInterfaceJIT::TimesTwo, 0), stringGPR);
    cont8Bit.link(&jit);

    failures.append(jit.branch32(JSInterfaceJIT::AboveOrEqual, stringGPR, JSInterfaceJIT::TrustedImm32(0x100)));
    jit.move(JSInterfaceJIT::TrustedImmPtr(vm->smallStrings.singleCharacterStrings()), indexGPR);
    jit.loadPtr(JSInterfaceJIT::BaseIndex(indexGPR, stringGPR, JSInterfaceJIT::ScalePtr, 0), stringGPR);
    jit.ret();

    failures.link(&jit);
    jit.move(JSInterfaceJIT::TrustedImm32(0), stringGPR);
    jit.ret();

    LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID, JITCompilationMustSucceed, true);
    return FINALIZE_CODE(patchBuffer, ("String get_by_val stub"));
}

static void stringCharLoad(SpecializedThunkJIT& jit, JSGlobalData* vm)
{
    // load string
    jit.loadJSStringArgument(*vm, SpecializedThunkJIT::ThisArgument, SpecializedThunkJIT::regT0);

    // Load string length to regT2, and start the process of loading the data pointer into regT0
    jit.load32(MacroAssembler::Address(SpecializedThunkJIT::regT0, ThunkHelpers::jsStringLengthOffset()), SpecializedThunkJIT::regT2);
    jit.loadPtr(MacroAssembler::Address(SpecializedThunkJIT::regT0, ThunkHelpers::jsStringValueOffset()), SpecializedThunkJIT::regT0);
    jit.appendFailure(jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0));

    // load index
    jit.loadInt32Argument(0, SpecializedThunkJIT::regT1); // regT1 contains the index

    // Do an unsigned compare to simultaneously filter negative indices as well as indices that are too large
    jit.appendFailure(jit.branch32(MacroAssembler::AboveOrEqual, SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT2));

    // Load the character
    SpecializedThunkJIT::JumpList is16Bit;
    SpecializedThunkJIT::JumpList cont8Bit;
    // Load the string flags
    jit.load32(MacroAssembler::Address(SpecializedThunkJIT::regT0, StringImpl::flagsOffset()), SpecializedThunkJIT::regT2);
    jit.loadPtr(MacroAssembler::Address(SpecializedThunkJIT::regT0, StringImpl::dataOffset()), SpecializedThunkJIT::regT0);
    is16Bit.append(jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT2, MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
    jit.load8(MacroAssembler::BaseIndex(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, MacroAssembler::TimesOne, 0), SpecializedThunkJIT::regT0);
    cont8Bit.append(jit.jump());
    is16Bit.link(&jit);
    jit.load16(MacroAssembler::BaseIndex(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, MacroAssembler::TimesTwo, 0), SpecializedThunkJIT::regT0);
    cont8Bit.link(&jit);
}

static void charToString(SpecializedThunkJIT& jit, JSGlobalData* vm, MacroAssembler::RegisterID src, MacroAssembler::RegisterID dst, MacroAssembler::RegisterID scratch)
{
    jit.appendFailure(jit.branch32(MacroAssembler::AboveOrEqual, src, MacroAssembler::TrustedImm32(0x100)));
    jit.move(MacroAssembler::TrustedImmPtr(vm->smallStrings.singleCharacterStrings()), scratch);
    jit.loadPtr(MacroAssembler::BaseIndex(scratch, src, MacroAssembler::ScalePtr, 0), dst);
    jit.appendFailure(jit.branchTestPtr(MacroAssembler::Zero, dst));
}

MacroAssemblerCodeRef charCodeAtThunkGenerator(JSGlobalData* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    stringCharLoad(jit, vm);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "charCodeAt");
}

MacroAssemblerCodeRef charAtThunkGenerator(JSGlobalData* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    stringCharLoad(jit, vm);
    charToString(jit, vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
    jit.returnJSCell(SpecializedThunkJIT::regT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "charAt");
}

MacroAssemblerCodeRef fromCharCodeThunkGenerator(JSGlobalData* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    // load char code
    jit.loadInt32Argument(0, SpecializedThunkJIT::regT0);
    charToString(jit, vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
    jit.returnJSCell(SpecializedThunkJIT::regT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "fromCharCode");
}

MacroAssemblerCodeRef sqrtThunkGenerator(JSGlobalData* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    if (!jit.supportsFloatingPointSqrt())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));

    jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
    jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "sqrt");
}


#define UnaryDoubleOpWrapper(function) function##Wrapper
enum MathThunkCallingConvention { };
typedef MathThunkCallingConvention(*MathThunk)(MathThunkCallingConvention);
    
#if CPU(X86_64) && COMPILER(GCC) && (PLATFORM(MAC) || OS(LINUX))

#define defineUnaryDoubleOpWrapper(function) \
    asm( \
        ".text\n" \
        ".globl " SYMBOL_STRING(function##Thunk) "\n" \
        HIDE_SYMBOL(function##Thunk) "\n" \
        SYMBOL_STRING(function##Thunk) ":" "\n" \
        "call " GLOBAL_REFERENCE(function) "\n" \
        "ret\n" \
    );\
    extern "C" { \
        MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
    } \
    static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;

#elif CPU(X86) && COMPILER(GCC) && (PLATFORM(MAC) || OS(LINUX))
#define defineUnaryDoubleOpWrapper(function) \
    asm( \
        ".text\n" \
        ".globl " SYMBOL_STRING(function##Thunk) "\n" \
        HIDE_SYMBOL(function##Thunk) "\n" \
        SYMBOL_STRING(function##Thunk) ":" "\n" \
        "subl $8, %esp\n" \
        "movsd %xmm0, (%esp) \n" \
        "call " GLOBAL_REFERENCE(function) "\n" \
        "fstpl (%esp) \n" \
        "movsd (%esp), %xmm0 \n" \
        "addl $8, %esp\n" \
        "ret\n" \
    );\
    extern "C" { \
        MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
    } \
    static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;

#elif CPU(ARM_THUMB2) && COMPILER(GCC) && PLATFORM(IOS)

#define defineUnaryDoubleOpWrapper(function) \
    asm( \
        ".text\n" \
        ".align 2\n" \
        ".globl " SYMBOL_STRING(function##Thunk) "\n" \
        HIDE_SYMBOL(function##Thunk) "\n" \
        ".thumb\n" \
        ".thumb_func " THUMB_FUNC_PARAM(function##Thunk) "\n" \
        SYMBOL_STRING(function##Thunk) ":" "\n" \
        "sub sp, sp, #16\n" \
        "str lr, [sp, #0]\n" \
        "vmov r0, r1, d0\n" \
        "blx " GLOBAL_REFERENCE(function) "\n" \
        "vmov d0, r0, r1\n" \
        "ldr lr, [sp, #0]\n" \
        "add sp, sp, #16\n" \
        "bx lr\n" \
    ); \
    extern "C" { \
        MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
    } \
    static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;

#elif CPU(ARM64)

#define defineUnaryDoubleOpWrapper(function) \
    asm( \
        ".text\n" \
        ".align 2\n" \
        ".globl " SYMBOL_STRING(function##Thunk) "\n" \
        HIDE_SYMBOL(function##Thunk) "\n" \
        SYMBOL_STRING(function##Thunk) ":" "\n" \
        "b " GLOBAL_REFERENCE(function) "\n" \
    ); \
    extern "C" { \
        MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
    } \
    static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;

#else

#define defineUnaryDoubleOpWrapper(function) \
    static MathThunk UnaryDoubleOpWrapper(function) = 0
#endif

defineUnaryDoubleOpWrapper(jsRound);
defineUnaryDoubleOpWrapper(exp);
defineUnaryDoubleOpWrapper(log);
defineUnaryDoubleOpWrapper(floor);
defineUnaryDoubleOpWrapper(ceil);
defineUnaryDoubleOpWrapper(trunc);

static const double halfConstant = 0.5;
    
MacroAssemblerCodeRef floorThunkGenerator(JSGlobalData* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    MacroAssembler::Jump nonIntJump;
    if (!UnaryDoubleOpWrapper(floor) || !jit.supportsFloatingPoint())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    nonIntJump.link(&jit);
    jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
#if CPU(ARM64)
    SpecializedThunkJIT::JumpList doubleResult;
    jit.floorDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
    jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    doubleResult.link(&jit);
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);
#else
    SpecializedThunkJIT::Jump intResult;
    SpecializedThunkJIT::JumpList doubleResult;
    if (jit.supportsFloatingPointTruncate()) {
        jit.moveZeroToDouble(SpecializedThunkJIT::fpRegT1);
        doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
        SpecializedThunkJIT::JumpList slowPath;
        // Handle the negative doubles in the slow path for now.
        slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
        slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0));
        intResult = jit.jump();
        slowPath.link(&jit);
    }
    jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(floor));
    jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
    if (jit.supportsFloatingPointTruncate())
        intResult.link(&jit);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    doubleResult.link(&jit);
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);
#endif // CPU(ARM64)
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "floor");
}

MacroAssemblerCodeRef ceilThunkGenerator(JSGlobalData* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    if (!UnaryDoubleOpWrapper(ceil) || !jit.supportsFloatingPoint())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    MacroAssembler::Jump nonIntJump;
    jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    nonIntJump.link(&jit);
    jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
#if CPU(ARM64)
    jit.ceilDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
#else
    jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(ceil));
#endif // CPU(ARM64)
    SpecializedThunkJIT::JumpList doubleResult;
    jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    doubleResult.link(&jit);
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "ceil");
}

MacroAssemblerCodeRef truncThunkGenerator(VM* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    if (!UnaryDoubleOpWrapper(trunc) || !jit.supportsFloatingPoint())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    MacroAssembler::Jump nonIntJump;
    jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    nonIntJump.link(&jit);
    jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
    if (jit.supportsFloatingPointRounding())
        jit.roundTowardZeroDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
    else
        jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(trunc));

    SpecializedThunkJIT::JumpList doubleResult;
    jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    doubleResult.link(&jit);
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "trunc");
}

MacroAssemblerCodeRef roundThunkGenerator(JSGlobalData* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    if (!UnaryDoubleOpWrapper(jsRound) || !jit.supportsFloatingPoint())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    MacroAssembler::Jump nonIntJump;
    jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    nonIntJump.link(&jit);
    jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
    SpecializedThunkJIT::Jump intResult;
    SpecializedThunkJIT::JumpList doubleResult;
    if (jit.supportsFloatingPointTruncate()) {
        jit.moveZeroToDouble(SpecializedThunkJIT::fpRegT1);
        doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
        SpecializedThunkJIT::JumpList slowPath;
        // Handle the negative doubles in the slow path for now.
        slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
        jit.loadDouble(MacroAssembler::TrustedImmPtr(&halfConstant), SpecializedThunkJIT::fpRegT1);
        jit.addDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
        slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0));
        intResult = jit.jump();
        slowPath.link(&jit);
    }
    jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(jsRound));
    jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
    if (jit.supportsFloatingPointTruncate())
        intResult.link(&jit);
    jit.returnInt32(SpecializedThunkJIT::regT0);
    doubleResult.link(&jit);
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "round");
}

MacroAssemblerCodeRef expThunkGenerator(JSGlobalData* vm)
{
    if (!UnaryDoubleOpWrapper(exp))
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    SpecializedThunkJIT jit(vm, 1);
    if (!jit.supportsFloatingPoint())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
    jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(exp));
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "exp");
}

MacroAssemblerCodeRef logThunkGenerator(JSGlobalData* vm)
{
    if (!UnaryDoubleOpWrapper(log))
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    SpecializedThunkJIT jit(vm, 1);
    if (!jit.supportsFloatingPoint())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
    jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(log));
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "log");
}

MacroAssemblerCodeRef absThunkGenerator(JSGlobalData* vm)
{
    SpecializedThunkJIT jit(vm, 1);
    if (!jit.supportsFloatingPointAbs())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
    MacroAssembler::Jump nonIntJump;
    jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
    jit.rshift32(SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(31), SpecializedThunkJIT::regT1);
    jit.add32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
    jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
    jit.appendFailure(jit.branch32(MacroAssembler::Equal, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1 << 31)));
    jit.returnInt32(SpecializedThunkJIT::regT0);
    nonIntJump.link(&jit);
    // Shame about the double int conversion here.
    jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
    jit.absDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
    jit.returnDouble(SpecializedThunkJIT::fpRegT1);
    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "abs");
}

MacroAssemblerCodeRef imulThunkGenerator(VM* vm)
{
    SpecializedThunkJIT jit(vm, 2);
    MacroAssembler::Jump nonIntArg0Jump;
    jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArg0Jump);
    SpecializedThunkJIT::Label doneLoadingArg0(&jit);
    MacroAssembler::Jump nonIntArg1Jump;
    jit.loadInt32Argument(1, SpecializedThunkJIT::regT1, nonIntArg1Jump);
    SpecializedThunkJIT::Label doneLoadingArg1(&jit);
    jit.mul32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
    jit.returnInt32(SpecializedThunkJIT::regT0);

    if (jit.supportsFloatingPointTruncate()) {
        nonIntArg0Jump.link(&jit);
        jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
        jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(doneLoadingArg0, &jit);
        jit.appendFailure(jit.jump());
    } else
        jit.appendFailure(nonIntArg0Jump);

    if (jit.supportsFloatingPointTruncate()) {
        nonIntArg1Jump.link(&jit);
        jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT1);
        jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT1, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(doneLoadingArg1, &jit);
        jit.appendFailure(jit.jump());
    } else
        jit.appendFailure(nonIntArg1Jump);

    return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "imul");
}

MacroAssemblerCodeRef randomThunkGenerator(VM* vm)
{
    SpecializedThunkJIT jit(vm, 0);
    if (!jit.supportsFloatingPoint())
        return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));

#if USE(JSVALUE64) && 0
    jit.emitRandomThunk(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT3, SpecializedThunkJIT::fpRegT0);
    jit.returnDouble(SpecializedThunkJIT::fpRegT0);

    return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "random");
#else
    return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
#endif
}

}

#endif // ENABLE(JIT)
