// gen-threaded.ci -- direct-threaded code generation and runtime -*- C++ -*-
//
// Author: Ian.Piumarta@inria.fr
//
// Last edited: Tue Mar  9 17:56:51 1999 by piumarta (Ian Piumarta) on yoko
		     
#include "interp.h"
#include "util.h"
#include "translate.h"
#include "error.h"
		     
#include "primitiveTable.decl"

#include "cache.h"

#include <string.h>
#include <stdio.h>
#include <stdlib.h>

#include"execute.h"

#ifdef GATHER_STATS
unsigned stat_icHits= 0, stat_icMisses= 0;
unsigned stat_mcHits= 0, stat_mcMisses= 0;
unsigned stat_acHits= 0, stat_acMisses= 0;
unsigned stat_jitterEntries= 0;
unsigned stat_linkedSends= 0;
unsigned stat_unlinkedSends= 0;
unsigned long long lastEntry= 0, lastExit= 0;
unsigned long long interpMSecs= 0, jitterMSecs= 0;
unsigned stat_normalActivations= 0, stat_quickActivations= 0;

# define IC_HIT()	++stat_icHits
# define IC_MISS()	++stat_icMisses
# define MC_HIT()	++stat_mcHits
# define MC_MISS()	++stat_mcMisses
# define AC_HIT()	++stat_acHits
# define AC_MISS()	++stat_acMisses
# define LINKED_SEND()	++stat_linkedSends
# define UNLINKED_SEND()	++stat_unlinkedSends
# define ACTIVATE_QK()	++stat_quickActivations
# define ACTIVATE()	++stat_normalActivations
#else
# define IC_HIT()
# define IC_MISS()
# define MC_HIT()
# define MC_MISS()
# define AC_HIT()
# define AC_MISS()
# define LINKED_SEND()
# define UNLINKED_SEND()
# define ACTIVATE_QK()
# define ACTIVATE()
#endif


///
/// THREADED CODE INTERPRETER
///


#define temporary(index)	(temporaryPointer[index])
#define literal(index)		(translatedMethod->compiledMethod()->literals[index])
#define receiver()		(temporary(-1))
		     
#include "opcodes.h"

#ifdef THREADED
void **opcodeTable;
#endif

#ifndef NDEBUG
bool externalRegisters= true;
#endif


void runJitter(bool initFlag)
{
#if defined(__i386__)
  register word		*localIP asm ("%esi");
  register oop		*localSP asm ("%ebx");
  word			*savedIP;
  InlineCache		*icbm;
# undef PREFETCH
#elif defined(PPC)
  register word		*localIP asm ("29");
  register oop		*localSP asm ("28");
  register word		*savedIP asm ("27");
  register InlineCache	*icbm	 asm ("26");
# define CACHE_CONSTS
# define PREFETCH
#elif defined(sparc)
  register word		*localIP asm ("%l0");
  register oop		*localSP asm ("%l1");
  register word		*savedIP asm ("%l2");
  register InlineCache	*icbm	 asm ("%l3");
# define CACHE_CONSTS
# define PREFETCH
#else
  register word		*localIP;
  register oop		*localSP;
  word			*savedIP;
  InlineCache		*icbm;
# if !defined(EPEND)
#   warning: REGISTERS NOT ALLOCATED
# endif
#endif

  oop *savedTP= 0;

# define qkReceiver temporaryPointer[-1]

  localIP= 0;
  localSP= 0;
  savedIP= 0;
  icbm=    0;

#ifdef CACHE_CONSTS
# define NIL_OOP nilOop
  register oop	 nilOop=   nilObj;
  register oop	 trueOop=  trueObj;
  register oop	 falseOop= falseObj;
#else
# define	 NIL_OOP   nilObj
# define	 trueOop   trueObj
# define	 falseOop  falseObj
#endif

#define	fetchWord()		(*localIP++)
#define	peekWord()		(*localIP)

#define	internalPush(object)	(*++localSP= (object))
#define	internalDrop(nItems)	(localSP-= (nItems))
#define	internalPop()		(*localSP--)
#define	internalUnPop(nItems)	(localSP+= (nItems))
#define	internalStackTop()	(*localSP)
#define	internalStackValue(idx)	(localSP[-(idx)])

#define	internalDrop1Push(obj)	(*localSP= (obj))
#define	internalDrop2Push(obj)	(*--localSP= (obj))
#define	internalDrop3Push(obj)	(*(localSP-= 2)= (obj))

#define	externalise()		assert(activeCachedContext != 0); \
				assert(!externalRegisters); \
				assert((externalRegisters= true)); \
				instructionPointer= localIP; \
				stackPointer= localSP; \
				assert(localSP >= (activeCachedContext->stack - 1))

#define	internalise()		assert(activeCachedContext != 0); \
				assert(externalRegisters); \
				assert(!(externalRegisters= false)); \
				localIP= instructionPointer; \
				localSP= stackPointer; \
				assert(localSP >= (activeCachedContext->stack - 1))
#ifdef THREADED
# include "bcTable.h"
# define CASE(N)	assert(false); case (N): op_##N: asm ("op_"#N":"); ++localIP;
# define NEXT()		goto *(void *)peekWord()
# ifdef PREFETCH
#   define LOAD()	register void *ir= (void *)peekWord()
#   define JUMP()	goto *ir
# else
#   define LOAD()
#   define JUMP()	NEXT()
# endif
#else
# define CASE(N)	assert(false); case (N): ++localIP;
# define NEXT()		break
# define LOAD()
# define JUMP()		NEXT()
# ifndef EPEND
#   warning: JUMP TABLE DISABLED
# endif
#endif

  if (initFlag) {
#ifdef THREADED
    opcodeTable= bc_tab;
#endif
    return;
  }

  activated= true;

  assert(activeCachedContext != 0);

  internalise();

  for (;;) {

#ifdef DEBUG
    {
      externalise();
      okayJitterObjects();
      internalise();
    }
#endif

    if (savedIP == 0) {
      noteExecutionState(receiver, translatedMethod, localIP,
			 activeCachedContext, localSP);
    } else {
      noteExecutionState(receiver, icbm->translatedMethod, localIP,
			 activeCachedContext, localSP);
    }

#ifdef THREADED
    NEXT();
#endif

    switch (peekWord()) {

    CASE(OpIllegal) {
      error("illegal opcode executed");
      NEXT();
    }

    // REGULAR OPCODES

    CASE(OpJmp) {
      localIP= (word *)peekWord();
      NEXT();
    }

    CASE(OpJmpChk) {
      localIP= (word *)peekWord();
      if (--interruptCheckCounter >= 0) NEXT();
      externalise();
      checkForInterrupts();
      if (activeCachedContext) {
	internalise();
	NEXT();
      }
      PRINTF(("return to Interpreter from interrupted loop"));
    }

    resumeActiveContext: {
      assert(externalRegisters);
      Context *ac= activeContext;
      ac->beRoot();
      fetchContextRegisters(ac);	// this might not be necessary!
      activeCachedContext= 0;
      topStableContext= nilObj->asContext();
      return;
    }

    CASE(OpJmpT) {
      oop top= internalPop();
      if (top == trueOop) localIP= (word *)peekWord();
      else {
	++localIP;
	if (top != falseOop) goto doMustBeBool;
      }
      NEXT();
    }

    CASE(OpJmpF) {
      oop top= internalPop();
      if (top == falseOop) localIP= (word *)peekWord();
      else {
	++localIP;
	if (top != trueOop) goto doMustBeBool;
      }
      NEXT();
    }

    doMustBeBool: {
      assert(!externalRegisters);
      internalUnPop(1);
      messageSelector= SelectorMustBeBoolean;
      argumentCount= 0;
      receiverClass= internalStackValue(argumentCount)->fetchClass();
      goto doSend;
    }

    CASE(OpLdSelf) {
      LOAD();
      internalPush(receiver);
      JUMP();
    }

    CASE(OpLdTrue) {
      LOAD();
      internalPush(trueOop);
      JUMP();
    }

    CASE(OpLdFalse) {
      LOAD();
      internalPush(falseOop);
      JUMP();
    }

    CASE(OpLdNil) {
      LOAD();
      internalPush(NIL_OOP);
      JUMP();
    }

    CASE(OpLdMinusOne) {
      LOAD();
      internalPush(Object::integer(-1));
      JUMP();
    }

    CASE(OpLdZero) {
      LOAD();
      internalPush(Object::integer(0));
      JUMP();
    }

    CASE(OpLdOne) {
      LOAD();
      internalPush(Object::integer(1));
      JUMP();
    }

    CASE(OpLdTwo) {
      LOAD();
      internalPush(Object::integer(2));
      JUMP();
    }

    CASE(OpLdInst) {
      register const int index= fetchWord();
      LOAD();
      internalPush(receiver->_oops[index]);
      JUMP();
    }

    CASE(OpLdTemp) {
      register const int index= fetchWord();
      LOAD();
      internalPush(temporary(index));
      JUMP();
    }

    CASE(OpLdLit) {
      register const int index= fetchWord();
      LOAD();
      internalPush(literal(index));
      JUMP();
    }

    CASE(OpLdLitInd) {
      register const int index= fetchWord();
      LOAD();
      internalPush(literal(index)->asAssociation()->value);
      JUMP();
    }

    CASE(OpLdThisContext) {
      CachedContext *acc= activeCachedContext;
      if (acc->pseudo == 0) {
	externalise();
	acc->allocatePseudoContext();
	internalise();
      }
      LOAD();
      internalPush(acc->pseudo);
      JUMP();
    }

    CASE(OpStInst) {
      register const int index= fetchWord();
      register oop rcvr= receiver;
      register oop top= internalStackTop();
      LOAD();
      rcvr->checkStore(rcvr->_oops[index]= top);
      JUMP();
    }

    CASE(OpStTemp) {
      register const int index= fetchWord();
      LOAD();
      temporary(index)= internalStackTop();
      JUMP();
    }

    CASE(OpStLitInd) {
      register const int index= fetchWord();
      LOAD();
      register oop assoc= literal(index);
      register oop value= internalStackTop();
      assoc->checkStore(assoc->asAssociation()->value= value);
      JUMP();
    }

    CASE(OpPopInst) {
      register const int index= fetchWord();
      LOAD();
      oop rcvr= receiver;
      oop top= internalStackTop();
      rcvr->checkStore(rcvr->_oops[index]= top);
      internalDrop(1);
      JUMP();
    }

    CASE(OpPopTemp) {
      register const int index= fetchWord();
      LOAD();
      temporary(index)= internalStackTop();
      internalDrop(1);
      JUMP();
    }

    CASE(OpPopLitInd) {
      register const int index= fetchWord();
      LOAD();
      oop assoc= literal(index);
      oop value= internalStackTop();
      assoc->checkStore(assoc->asAssociation()->value= value);
      internalDrop(1);
      NEXT();
    }

#define nonLocalReturn(result) { \
      returnResult= (result); \
      if (activeCachedContext->isMethod()) goto doLocalReturn; \
      goto doNonLocalReturn; \
    }

    CASE(OpRetSelf) {
      nonLocalReturn(receiver);
      NEXT();
    }

    CASE(OpRetNil) {
      nonLocalReturn(NIL_OOP);
    }

    doNonLocalReturn: {
      assert(!externalRegisters);
#     ifndef CACHE_CONSTS
      oop nilOop= nilObj;
#     endif
      CachedContext *acc= activeCachedContext;
      if (acc->pseudo) {
	acc->sp= localSP;
	acc->ip= localIP;
      }
      Context *home= acc->closure->home->asContext();
      if (home->isPseudo()) {
        CachedContext *phc= home->asPseudoContext()->cachedContext();
	if (phc == lowestCachedContext && topStableContext == nilOop)
	  goto cannotReturn;
        // unwind stack to home context
        do {
          acc->unlink(nilOop->asContext());
          acc= acc->parent();
        } while (acc != phc);
        activeCachedContext= acc;
	if (acc->pseudo) {
	  // otherwise we risk stomping a huge SP value
	  // into acc in the pseudo check of doLocalReturn
	  localIP= acc->ip;
	  localSP= acc->sp;
	}
        goto doLocalReturn;
      }
      // non-local return to Interpreter: this is EXTREMELY rare
      if (home == nilOop) goto cannotReturn;
      // check that we can reach the home context
      Context *stable= topStableContext;
      while (stable != home) {
	if (stable == nilOop) goto cannotReturn;
	stable= stable->sender;
      }
      // check that we can return into home's sender
      stable= home->sender;
      if (stable == nilOop || stable->pc == nilOop) goto cannotReturn;
      // stabilise context cache
      assert(acc == activeCachedContext);
      acc->unlink(nilOop->asContext());
      while (acc != lowestCachedContext)
	(acc= acc->parent())->unlink(nilOop->asContext());
      // unlink stable contexts up to home
      stable= topStableContext;
      while (stable != home) {
	Context *ctx= stable;
	stable= stable->sender;
	ctx->unlink(nilOop);
      }
      // return
      activeContext= stable->sender;
      stable->unlink();
      // registers are now effectively external
      assert(externalRegisters= true);
      goto returnToActiveContext;
    }

    CASE(OpRetTrue) {
      nonLocalReturn(trueOop);
      NEXT();
    }

    CASE(OpRetFalse) {
      nonLocalReturn(falseOop);
      NEXT();
    }

    CASE(OpMethodRet) {
      returnResult= internalStackTop();
      if (activeCachedContext->isBlock()) goto doNonLocalReturn;
    }
    // FALL THROUGH...

    doLocalReturn: {
      assert(!externalRegisters);
      CachedContext *cx= activeCachedContext;
      if (cx->pseudo) {
	cx->sp= localSP;
	cx->ip= localIP;
	cx->unlink(NIL_OOP->asContext());
      }
      if (cx != lowestCachedContext) {
        activeCachedContext= cx= cx->parent();
        internalFetchCachedContextRegisters(cx);
        internalPush(returnResult);
        if (--interruptCheckCounter < 0) {
	  externalise();
          checkForInterrupts();
	  if (activeCachedContext == 0) {
	    PRINTF(("return to Interpreter from interrupted return"));
	    goto resumeActiveContext;
	  }
	  internalise();
	}
        noteReturn(receiver, translatedMethod, localIP,
                   activeCachedContext, localSP);
	assert(translatedMethod == activeCachedContext->translatedMethod);
        NEXT();
      }
      if ((topStableContext == nilObj) || (topStableContext->pc == nilObj))
	goto cannotReturn;
      externalise();
      activeContext= topStableContext;
      // FALL THROUGH...
    }

    returnToActiveContext: {
      assert(externalRegisters);
      Context *ac= activeContext;
      ac->beRoot();
      fetchContextRegisters(ac);
      push(returnResult);
      PRINTF(("return to Interpreter from empty context cache\n"));
      PRINTF(("top of stack: %p\n", stackValue(0)));
#     ifdef TRACE
      dumpObject("top of stack: ", stackValue(0));
#     endif
      activeCachedContext= 0;
      topStableContext= nilObj->asContext();
      return;
    }

    cannotReturn: {
      assert(!externalRegisters);
      CachedContext *acc= activeCachedContext;
      internalPush(acc->allocatePseudoContext());
      internalPush(returnResult);
      messageSelector= SelectorCannotReturn;
      argumentCount= 1;
      receiverClass= internalStackValue(argumentCount)->fetchClass();
      goto doSend;
    }

    CASE(OpBlockRet) {
      returnResult= internalStackTop();
      goto doLocalReturn;
      NEXT();
    }

    CASE(OpSend)
    doLink: {
      assert(!externalRegisters);
      const word descriptor= fetchWord();
      ++localIP; // linked method
      argumentCount= (descriptor >> 16);
      const word selIndex= descriptor & 0xff;
      // this is a lousy encoding
      if (argumentCount == 255) {
	messageSelector= SpecialSelectors->at((selIndex)*2);
	argumentCount= SpecialSelectors->at((selIndex)*2 + 1)->integerValue();
      } else {
	messageSelector= literal(descriptor & 0xff);
      }
      receiverClass= internalStackValue(argumentCount)->fetchClass();

#ifndef LINK_SENDS
      goto doSend;
#endif

#ifdef TRACE
      printf("OpSend: ");
      printNameOfClasscount(receiverClass, 5);
      printf(">>");
      printStringOf(messageSelector);
      printf("\n");
#endif

      externalise();
      // LINK SEND SITE
      assert(instructionPointer[-3] == OP(OpSend) ||
	     instructionPointer[-3] == OP(OpAt) ||
	     instructionPointer[-3] == OP(OpAtPut));
      if (receiverClass == ClassPseudoContext) goto executePseudoReceiver;
      if (!lookupInMethodCacheSelclass(messageSelector, receiverClass)) {
	MC_MISS();
	//printStringOf(messageSelector);
	//dumpObject(" in ", receiverClass);
	lookupMethodInClass(receiverClass);
	addToMethodCacheSelclassmethodprimIndex(messageSelector,
						receiverClass,
						newMethod,
						primitiveIndex);
      } else
	MC_HIT();
      TranslatedMethod *tm=
	translatedMethodFor(newMethod, stackValue(argumentCount));
      icbm= tm->inlineCacheFor(receiverClass, false);
      assert(receiverClass == icbm->receiverClass());
      internalise();
      if (messageSelector != SelectorDoesNotUnderstand) {
	localIP[-3]= OP(OpLinkedSend);
	localIP[-1]= (word)icbm;
      }
      savedIP= localIP;
      localIP= (word *)icbm->translatedMethod->uncheckedEntry();
      NEXT();
    }

    CASE(OpSuper) {
      word descriptor= fetchWord();
      ++localIP; // linked method
      receiverClass=
	methodClassOf(translatedMethod->compiledMethod())->superclass;
      messageSelector= literal(descriptor & 0xff);
      argumentCount= (descriptor >> 16);

#ifndef LINK_SENDS
      goto doSend;
#endif

#ifdef TRACE
      printf("OpSuper: ");
      printNameOfClasscount(receiverClass, 5);
      printf(">>");
      printStringOf(messageSelector);
      printf("\n");
#endif

      externalise();
      // LINK SEND SITE
      assert(instructionPointer[-3] == OP(OpSuper));
      if (!lookupInMethodCacheSelclass(messageSelector, receiverClass)) {
	MC_MISS();
	lookupMethodInClass(receiverClass);
	addToMethodCacheSelclassmethodprimIndex(messageSelector,
						receiverClass,
						newMethod,
						primitiveIndex);
      } else
	MC_HIT();
      TranslatedMethod *tm=
	translatedMethodFor(newMethod, stackValue(argumentCount));
      icbm= tm->inlineCacheFor(receiverClass, true);
      assert(receiverClass == icbm->receiverClass());
      internalise();
      if (messageSelector != SelectorDoesNotUnderstand) {
	localIP[-3]= OP(OpLinkedSuper);
	localIP[-1]= (word)icbm;
      }
      savedIP= localIP;
      localIP= (word *)icbm->translatedMethod->uncheckedEntry();
      NEXT();
    }

    doSend: {
      assert(!externalRegisters);
      UNLINKED_SEND();
      externalise();
      if (lookupInMethodCacheSelclass(messageSelector, receiverClass)) {
	MC_HIT();
	executeNewMethod();
	if (activeCachedContext) {
	  internalise();
	  NEXT();
	}
	PRINTF(("return to Interpreter from primitiveResponse(1)...\n"));
	goto resumeActiveContext;
      }
      if (receiverClass != ClassPseudoContext) {
	MC_MISS();
	lookupMethodInClass(receiverClass);
	addToMethodCacheSelclassmethodprimIndex(messageSelector, receiverClass,
						newMethod, primitiveIndex);
	executeNewMethod();
	if (activeCachedContext) {
	  internalise();
	  NEXT();
	}
	PRINTF(("return to Interpreter from primitiveResponse(2)...\n"));
	goto resumeActiveContext;
      }
      // FALL THROUGH...
    }
    // PSEUDO RECEIVER
    executePseudoReceiver: {
      assert(externalRegisters);
      Context *ac= activeContext= flushContextCache();
      ac->beRoot();
      fetchContextRegisters(ac);
      receiverClass= stackValue(argumentCount)->fetchClass();
      assert(receiverClass == ClassMethodContext ||
	     receiverClass == ClassBlockContext);
      if (!lookupInMethodCacheSelclass(messageSelector, receiverClass)) {
	MC_MISS();
	lookupMethodInClass(receiverClass);
	addToMethodCacheSelclassmethodprimIndex(messageSelector,
						receiverClass,
						newMethod,
						primitiveIndex);
      } else {
	MC_HIT();
      }
      bytecodeExecuteNewMethod();
      PRINTF(("return to Interpreter from primitiveResponse(3)...\n"));
      goto resumeActiveContext;
    }

    CASE(OpPop) {
      LOAD();
      internalDrop(1);
      JUMP();
    }

    CASE(OpDup) {
      LOAD();
      oop top= internalStackTop();
      internalPush(top);
      JUMP();
    }

    // SPECIAL SENDS

#define	specialSend(selIndex) { \
      messageSelector= SpecialSelectors->at((selIndex)*2); \
      argumentCount= SpecialSelectors->at((selIndex)*2 + 1)->integerValue(); \
      receiverClass= internalStackValue(argumentCount)->fetchClass(); \
      goto doSend; \
    }

    CASE(OpAdd) {
      oop rhs= internalStackValue(0);
      oop lhs= internalStackValue(1);
      if (areIntegerObjects(lhs, rhs)) {
	int result= lhs->integerValue() + rhs->integerValue();
	if (Object::isIntegerValue(result)) {
	  LOAD();
	  internalDrop2Push(Object::integer(result));
	  JUMP();
	}
      }
      externalise();
      successFlag= 1;
      primitiveFloatAdd();
      if (!successFlag) {
	successFlag= 1;
	primitiveAdd();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(0);
      NEXT();
    }

    CASE(OpSubtract) {
      int rhs= (int)internalStackValue(0);
      int lhs= (int)internalStackValue(1);
      if (areIntegerObjects(lhs, rhs)) {
	int result= (lhs >> 1) - (rhs >> 1);
	if (Object::isIntegerValue(result)) {
	  LOAD();
	  internalDrop2Push(Object::integer(result));
	  JUMP();
	}
      }
      externalise();
      successFlag= 1;
      primitiveFloatSubtract();
      if (!successFlag) {
	successFlag= 1;
	primitiveSubtract();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(1);
      NEXT();
    }

    CASE(OpLess) {
      int lhs= (int)internalStackValue(1);
      int rhs= (int)internalStackValue(0);
      if (areIntegerObjects(lhs, rhs)) {
	LOAD();
	internalDrop2Push((lhs < rhs) ? trueOop : falseOop);
	JUMP();
      }
      externalise();
      successFlag= 1;
      primitiveFloatLessThan();
      if (!successFlag) {
	successFlag= 1;
	primitiveLessThan();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(2);
      NEXT();
    }

    CASE(OpGreater) {
      int lhs= (int)internalStackValue(1);
      int rhs= (int)internalStackValue(0);
      if (areIntegerObjects(lhs, rhs)) {
	LOAD();
	internalDrop2Push((lhs > rhs) ? trueOop : falseOop);
	JUMP();
      }
      externalise();
      successFlag= 1;
      primitiveFloatGreaterThan();
      if (!successFlag) {
	successFlag= 1;
	primitiveGreaterThan();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(3);
      NEXT();
    }

    CASE(OpLessEqual) {
      int lhs= (int)internalStackValue(1);
      int rhs= (int)internalStackValue(0);
      if (areIntegerObjects(lhs, rhs)) {
	LOAD();
	internalDrop2Push((lhs <= rhs) ? trueOop : falseOop);
	JUMP();
      }
      externalise();
      successFlag= 1;
      primitiveFloatLessOrEqual();
      if (!successFlag) {
	successFlag= 1;
	primitiveLessOrEqual();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(4);
      NEXT();
    }

    CASE(OpGreaterEqual) {
      int lhs= (int)internalStackValue(1);
      int rhs= (int)internalStackValue(0);
      if (areIntegerObjects(lhs, rhs)) {
	LOAD();
	internalDrop2Push((lhs >= rhs) ? trueOop : falseOop);
	JUMP();
      }
      externalise();
      successFlag= 1;
      primitiveFloatGreaterOrEqual();
      if (!successFlag) {
	successFlag= 1;
	primitiveGreaterOrEqual();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(5);
      NEXT();
    }

    CASE(OpEqual) {
      int lhs= (int)internalStackValue(1);
      int rhs= (int)internalStackValue(0);
      if (areIntegerObjects(lhs, rhs)) {
	LOAD();
	internalDrop2Push((lhs == rhs) ? trueOop : falseOop);
	JUMP();
      }
      externalise();
      successFlag= 1;
      primitiveFloatEqual();
      if (!successFlag) {
	successFlag= 1;
	primitiveEqual();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(6);
      NEXT();
    }

    CASE(OpNotEqual) {
      int lhs= (int)internalStackValue(1);
      int rhs= (int)internalStackValue(0);
      if (areIntegerObjects(lhs, rhs)) {
	LOAD();
	internalDrop2Push((lhs != rhs) ? trueOop : falseOop);
	JUMP();
      }
      externalise();
      successFlag= 1;
      primitiveFloatNotEqual();
      if (!successFlag) {
	successFlag= 1;
	primitiveNotEqual();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(7);
      NEXT();
    }

    CASE(OpMultiply) {
      int rcvr;
      int arg;
      int result;

      rcvr= (int)internalStackValue(1);
      arg= (int)internalStackValue(0);
      if (areIntegerObjects(rcvr, arg)) {
	rcvr= integerValueOf(rcvr);
	arg= integerValueOf(arg);
	result= rcvr * arg;
	if (((arg == 0) || ((result / arg) == rcvr)) &&
	    (Object::isIntegerValue(result))) {
	  LOAD();
	  internalDrop2Push(Object::integer(result));
	  JUMP();
	}
	externalise();
	successFlag= 0;
      } else {
	externalise();
	successFlag= 1;
	primitiveFloatMultiply();
      }
      if (!successFlag) {
	successFlag= 1;
	primitiveMultiply();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(8);
      NEXT();
    }

    CASE(OpDivide) {
      int rcvr= (int)internalStackValue(1);
      int arg= (int)internalStackValue(0);
      if (areIntegerObjects(rcvr, arg)) {
	rcvr= integerValueOf(rcvr);
	arg= integerValueOf(arg);
	if ((arg != 0) && ((rcvr % arg) == 0)) {
	  int result= rcvr / arg;
	  if (Object::isIntegerValue(result)) {
	    LOAD();
	    internalDrop2Push(Object::integer(result));
	    JUMP();
	  }
	}
	externalise();
	successFlag= 0;
      } else {
	externalise();
	successFlag= 1;
	primitiveFloatDivide();
      }
      if (!successFlag) {
	successFlag= 1;
	primitiveDivide();
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(9);
      NEXT();
    }

    CASE(OpMod) {
      successFlag= 1;
      int quotient= doPrimitiveModby(internalStackValue(1),
				     internalStackValue(0));
      if (successFlag) {
	LOAD();
	internalDrop2Push(Object::integer(quotient));
	JUMP();
      }
      specialSend(10);
      NEXT();
    }

# define simplePrim(index, primitiveFunction) { \
      externalise(); \
      primitiveFunction(); \
      internalise(); \
      if (successFlag) NEXT(); \
      specialSend(index); \
    }

    CASE(OpMakePoint)	simplePrim(11, primitiveMakePoint); NEXT();
    CASE(OpBitShift)	simplePrim(12, primitiveBitShift); NEXT();

    CASE(OpDiv) {
      successFlag= 1;
      int quotient= doPrimitiveDivby(internalStackValue(1),
				     internalStackValue(0));
      if (successFlag) {
	LOAD();
	internalDrop2Push(Object::integer(quotient));
	JUMP();
      }
      specialSend(13);
      NEXT();
    }

    CASE(OpBitAnd)	simplePrim(14, primitiveBitAnd); NEXT();
    CASE(OpBitOr)	simplePrim(15, primitiveBitOr); NEXT();

    CASE(OpAt) {
      const oop index= internalStackTop();
      const oop rcvr=  internalStackValue(1);
      if (index->isInteger()) {
	const int line= (word)rcvr & AtCacheMask;
	if (atCache[line + AtCacheOop] == (word)rcvr) {
# ifdef GATHER_STATS
	  ++stat_acHits;
# endif
	  successFlag= true;
	  const oop result=
	    commonVariableatcacheIndex(rcvr, index->integerValue(), line);
	  if (successFlag) {
	    LOAD();
	    internalDrop2Push(result);
	    JUMP();
	  }
	}
      }
# ifdef GATHER_STATS
      ++stat_acMisses;
# endif
      // convince primitiveAt to put us in the cache
      lkupClass= rcvr->fetchClass();
      specialSend(16);
      NEXT();
    }

    CASE(OpAtPut) {
      const oop value= internalStackTop();
      const oop index= internalStackValue(1);
      const oop rcvr=  internalStackValue(2);
      if ((!rcvr->isInteger()) && (index->isInteger())) {
	const int line= ((word)rcvr & AtCacheMask) + AtPutBase;
	if (atCache[line + AtCacheOop] == (word)rcvr) {
# ifdef GATHER_STATS
	  ++stat_acHits;
# endif
	  successFlag= true;
	  commonVariableatputcacheIndex(rcvr, index->integerValue(),
					value, line);
	  if (successFlag) {
	    LOAD();
	    internalDrop3Push(value);
	    JUMP();
	  }
	}
      }
# ifdef GATHER_STATS
      ++stat_acMisses;
# endif
      // convince primitiveAtPut to put us in the cache
      lkupClass= rcvr->fetchClass();
      specialSend(17);
      NEXT();
    }

    CASE(OpSize) {
      Class *arrayClass= internalStackValue(0)->fetchClass();
      externalise();
      successFlag= 1;
      if (okStreamArrayClass(arrayClass)) {
	primitiveSize();
      } else {
	successFlag= 0;
      }
      internalise();
      if (successFlag) NEXT();
      specialSend(18);
      NEXT();
    }

    CASE(OpNext)		specialSend(19); NEXT();
    CASE(OpNextPut)		specialSend(20); NEXT();
    CASE(OpAtEnd)		specialSend(21); NEXT();

    CASE(OpEquivalent) {
      // must test stack vals before stack pop
      LOAD();
      bool cond= (internalStackValue(1) == internalStackValue(0));
      internalDrop2Push(cond ? trueOop : falseOop);
      JUMP();
    }

    CASE(OpClass) {
      LOAD();
      internalDrop1Push(internalStackTop()->fetchClass());
      JUMP();
    }

    CASE(OpBlockCopy) {
      Class *rcvrClass= internalStackValue(1)->fetchClass();
      successFlag= ((rcvrClass == ClassBlockContext) ||
		    (rcvrClass == ClassMethodContext) ||
		    (rcvrClass == ClassPseudoContext));
      if (successFlag) {
	externalise();
	primitiveBlockCopy();
	internalise();
      }
      if (successFlag) NEXT();
      specialSend(24);
      NEXT();
    }

    CASE(OpValue) {
      oop block= internalStackTop();
      successFlag= 1;
      argumentCount= 0;
      successFlag= block->isBlockContext();
      if (successFlag) {
	externalise();
	primitiveValue();
	internalise();
      }
      if (successFlag) NEXT();
      specialSend(25);
      NEXT();
    }

    CASE(OpValueWithArg) {
      oop block= internalStackValue(1);
      successFlag= 1;
      argumentCount= 1;
      successFlag= block->isBlockContext();
      if (successFlag) {
	externalise();
	primitiveValue();
	internalise();
      }
      if (successFlag) NEXT();
      specialSend(26);
      NEXT();
    }

    CASE(OpDo)		specialSend(27); NEXT();
    CASE(OpNew)		specialSend(28); NEXT();
    CASE(OpNewWithArg)	specialSend(29); NEXT();
    CASE(OpPointX)	simplePrim(30, primitivePointX); NEXT();
    CASE(OpPointY)	simplePrim(31, primitivePointY); NEXT();

    // PROLOGUE OPCODES

    // quick push const methods

    CASE(OpPrimitivePushSelf) {
      // no-op
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      NEXT();
    }

    CASE(OpPrimitivePushTrue) {
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      LOAD();
      internalDrop1Push(trueObj);
      JUMP();
    }

    CASE(OpPrimitivePushFalse) {
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      LOAD();
      internalDrop1Push(falseObj);
      JUMP();
    }

    CASE(OpPrimitivePushNil) {
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      LOAD();
      internalDrop1Push(NIL_OOP);
      JUMP();
    }

    CASE(OpPrimitivePushMinusOne) {
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      LOAD();
      internalDrop1Push(Object::integer(-1));
      JUMP();
    }

    CASE(OpPrimitivePushZero) {
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      LOAD();
      internalDrop1Push(Object::integer(0));
      JUMP();
    }

    CASE(OpPrimitivePushOne) {
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      LOAD();
      internalDrop1Push(Object::integer(1));
      JUMP();
    }

    CASE(OpPrimitivePushTwo) {
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      LOAD();
      internalDrop1Push(Object::integer(2));
      JUMP();
    }

    // quick push inst var methods

    CASE(OpPrimitiveLoadInstVar) {
      const int index= localIP[0];
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      LOAD();
      const oop rcvr= internalStackTop();
      internalDrop1Push(rcvr->_oops[index]);
      JUMP();
    }

    // primitive methods

    CASE(OpPrimitive) {
      word primIndex= fetchWord();
      PRINTF(("OpPrimitive %d\n", primIndex));
      word *thisIP= localIP;
      word *sendIP= savedIP;
      localIP= savedIP;
#ifdef TRACE
      savedIP= 0;
#endif
      externalise();
      assert(stackPointer == localSP);
      argumentCount= icbm->argCount;
      primitiveIndex= primIndex;
      // THIS IS ONLY NEEDED FOR THE SOON-TO-BE-REMOVED primStoreInstVars
      //newMethod= icbm->compiledMethod();
      bool ok= primitiveResponse();
      if (activeCachedContext == 0) goto resumeActiveContext;
      assert(ok || (stackPointer == localSP));
      internalise();
      {
	LOAD();
	if (ok) {
	  PRINTF(("->success\n"));
	  // resume at localIP, already restored from savedIP
#ifdef TRACE
	  savedIP= 0;
#endif
	  JUMP();
	}
      }
      PRINTF(("->fail\n"));
      // resume to activate fail code
      localIP= thisIP;
      LOAD();
      savedIP= sendIP;
      if (*localIP == OP(OpQkActivate)) {
	printf("non-activated primitive failure: %d\n", primIndex);
      }
      JUMP();
    }

    // activated methods

    CASE(OpActivate) {
      ACTIVATE();
      const word numArgs= fetchWord();
      PRINTF(("activate %d\n", numArgs));
      // ACTIVATE
      CachedContext *oldContext= activeCachedContext;
      assert(oldContext != 0);

      CachedContext *acc= oldContext->child();
      if (acc == lowestCachedContext) {
	word *thisIP= localIP;
	localIP= savedIP;
	externalise();
	flushLowestCachedContext();
	internalise();
	localIP= thisIP;
      }
      {
	oop *frame= localSP - numArgs;	// location of receiver
	acc->stack= frame + 1;
	acc->receiver= receiver= *frame;	// could suppress this field
	oldContext->sp= frame - 1;
	assert(oldContext->sp >= (oldContext->stack - 1));
	oldContext->ip= savedIP;
      }

      LOAD();

      acc->pseudo=   0;
      acc->closure=  0;
      translatedMethod= icbm->translatedMethod;
      acc->translatedMethod= translatedMethod;
#ifdef USE_METHOD
      method= translatedMethod->compiledMethod();
#endif

      temporaryPointer= acc->stack;
      activeCachedContext= acc;
      assert(localSP == acc->stack + numArgs - PreInc);

#ifdef TRACE
      savedIP= 0;
#endif

      noteActivation(receiver, translatedMethod,
		     localIP, acc, localSP);

      JUMP();
    }

    CASE(OpReserve) {
      word numTemps= fetchWord();
      PRINTF(("OpReserve %d\n", numTemps));
      LOAD();
#ifndef CACHE_CONSTS
      register oop nilOop= nilObj;
#endif
      for (int i= numTemps; i != 0; --i)
	internalPush(nilOop);
      JUMP();
    }

    default: {
      error("unknown opcode executed");
      NEXT();
    }

    // LINKED SENDS

    CASE(OpLinkedSend) {
      LINKED_SEND();
      PRINTF(("OpLinkedSend\n"));
      ++localIP; // send descriptor
      icbm= (InlineCache *)fetchWord();
#ifdef TRACE
      printf("SEND entry:       %p\n",	icbm->entry);
      printf("classIndex:       %d:%d",	icbm->classIndex.major,
					icbm->classIndex.minor);
      printf(" -> ");
      printNameOfClasscount(classMemoizer->classAt(icbm->classIndex), 5);
      printf("\n");
      printf("ccIndex           %p\n",	(void *)icbm->ccIndex);
      printf("translatedMethod: %p\n",	icbm->translatedMethod);
      printf("nextCache         %p\n",	icbm->nextCache);
#endif
      savedIP= localIP;
      localIP= (word *)icbm->entry;
      NEXT();
    }

    CASE(OpLinkedSuper) {
      PRINTF(("OpLinkedSuper\n"));
      ++localIP; // send descriptor
      icbm= (InlineCache *)fetchWord();
#ifdef TRACE
      printf("SUPER entry:      %p\n",	icbm->entry);
      printf("classIndex:       %d:%d",	icbm->classIndex.major,
					icbm->classIndex.minor);
      printf(" -> ");
      printNameOfClasscount(classMemoizer->classAt(icbm->classIndex), 5);
      printf("\n");
      printf("ccIndex           %p\n",	(void *)icbm->ccIndex);
      printf("translatedMethod: %p\n",	icbm->translatedMethod);
      printf("nextCache         %p\n",	icbm->nextCache);
#endif
      savedIP= localIP;
      localIP= (word *)icbm->entry;
      NEXT();
    }

    CASE(OpCheckClass) {
      PRINTF(("OpCheckClass\n"));
      oop rcvr= internalStackValue(icbm->argCount);
      if (!rcvr->isInteger()) {
	if (classMemoizer->classAt(icbm->classIndex) == rcvr->fetchClass()) {
	  IC_HIT();
	  localIP+= TranslatedMethod::ClassEntrySkip;
	  NEXT();
	}
      }
    classMiss:
      PRINTF(("OpCheckClass: FAILED\n"));
    }

    doRelink: {
      IC_MISS();
      localIP= savedIP - 3;
#ifdef TRACE
      savedIP= 0;
#endif
      const int argSel= localIP[1];
      const int arg= (argSel >> 16);
      // check for special sends
      if (arg == 255) {
	const int sel= argSel & 0xff;
	if (sel == 16) { localIP[0]= OP(OpAt);	  NEXT(); }	// at:
	if (sel == 17) { localIP[0]= OP(OpAtPut); NEXT(); }	// at:put:
      }
      if (localIP[0] == OP(OpLinkedSend))
	localIP[0]= OP(OpSend);
      else if (localIP[0] == OP(OpLinkedSuper))
	localIP[0]= OP(OpSuper);
      else {
	error("could not relink for linked op 0x%08x (%d)",
	      (unsigned)localIP[0], (unsigned)localIP[0]);
      }
      NEXT();
    }

    CASE(OpCheckCompact) {
      PRINTF(("OpCheckCompact\n"));
      assert(icbm->ccIndex != 0);
      oop rcvr= internalStackValue(icbm->argCount);
      if (!rcvr->isInteger()) {
	if (icbm->ccIndex == internalStackValue(icbm->argCount)->ccBits()) {
	  IC_HIT();
	  localIP+= TranslatedMethod::CompactEntrySkip;
	  NEXT();
	}
      }
    compactMiss:
      PRINTF(("OpCheckCompact: FAILED\n"));
      goto doRelink;
    }

    CASE(OpCheckImmediate) {
      PRINTF(("OpCheckImmediate\n"));
      if (internalStackValue(icbm->argCount)->isInteger()) {
	IC_HIT();
	localIP+= TranslatedMethod::ImmediateEntrySkip;
	NEXT();
      }
      PRINTF(("OpCheckImmediate: FAILED\n"));
      goto doRelink;
    }

    CASE(OpRelink) {
      printf("OpRelink\n");
      // see the comment in TranslatedMethod::invalidate()
      methodMemoizer->remove(icbm->translatedMethod->memoIndex);
      goto doRelink;
    }

    // THIS SHOULD BE REMOVED ASAP!

    CASE(OpCheckImmediate_0_0) {
      PRINTF(("OpCheckImmediate\n"));
      if (internalStackValue(icbm->argCount)->isInteger()) {
	IC_HIT();
	// ACTIVATE
	CachedContext *oldContext= activeCachedContext;
	assert(oldContext != 0);

	CachedContext *acc= oldContext->child();
	if (acc == lowestCachedContext) {
	  word *thisIP= localIP;
	  localIP= savedIP;
	  externalise();
	  flushLowestCachedContext();
	  internalise();
	  localIP= thisIP;
	}
	acc->stack= localSP + 1;
	acc->receiver= receiver= *localSP;	// could suppress this field
	oldContext->sp= localSP - 1;
	assert(oldContext->sp >= (oldContext->stack - 1));
	oldContext->ip= savedIP;

	assert(*localIP == OP(OpActivate));

	localIP+= 2;				// skip OpActivate

	LOAD();

	acc->pseudo=   0;
	acc->closure=  0;
	translatedMethod= icbm->translatedMethod;
	acc->translatedMethod= translatedMethod;
#ifdef USE_METHOD
	method= translatedMethod->compiledMethod();
#endif

	temporaryPointer= acc->stack;
	activeCachedContext= acc;
	assert(localSP == acc->stack - PreInc);

#ifdef TRACE
	savedIP= 0;
#endif
	noteActivation(receiver, translatedMethod,
		       localIP, acc, localSP);

	JUMP();
      }
      PRINTF(("OpCheckImmediate_0_0: FAILED\n"));
      goto doRelink;
    }

    // SYNTHETIC QUICK METHODS

    CASE(OpQkActivate) {
      ACTIVATE_QK();
      assert(localSP > activeCachedContext->stack);
      const word numArgs= fetchWord();
#ifdef TRACE
      printf("OpQkActivate %d\n", numArgs);
#endif
      // ACTIVATE
      assert(icbm != 0);
      //assert(icbm->translatedMethod->activatedEntry == localIP);
      LOAD();
      activated= false;
      savedTP= temporaryPointer;
      temporaryPointer= localSP - numArgs + 1;
      assert(temporaryPointer > activeCachedContext->stack);
      JUMP();
    }

    CASE(OpQkRetSelf) {
#ifdef TRACE
      printf("OpQkRetSelf\n");
#endif
      if (!activated) {
	localIP= savedIP;
#ifdef TRACE
	savedIP= 0
#endif
	LOAD();
	oop result= qkReceiver;
	localSP= temporaryPointer - 1;
	temporaryPointer= savedTP;
	assert(*localSP == result);
	internalDrop1Push(result);
	JUMP();
      }
      printf("NLR: self\n");
      nonLocalReturn(receiver);
      NEXT();
    }

    CASE(OpQkRetTrue) {
#ifdef TRACE
      printf("OpQkRetTrue\n");
#endif
      if (!activated) {
	localIP= savedIP;
#ifdef TRACE
	savedIP= 0
#endif
	LOAD();
	localSP= temporaryPointer - 1;
	temporaryPointer= savedTP;
	internalDrop1Push(trueOop);
	JUMP();
      }
      nonLocalReturn(trueOop);
      NEXT();
    }

    CASE(OpQkRetFalse) {
#ifdef TRACE
      printf("OpQkRetFalse\n");
#endif
      if (!activated) {
	localIP= savedIP;
#ifdef TRACE
	savedIP= 0
#endif
	LOAD();
	localSP= temporaryPointer - 1;
	temporaryPointer= savedTP;
	internalDrop1Push(falseOop);
	JUMP();
      }
      nonLocalReturn(falseOop);
      NEXT();
    }

    CASE(OpQkRetNil) {
#ifdef TRACE
      printf("OpQkRetNil\n");
#endif
      if (!activated) {
	localIP= savedIP;
#ifdef TRACE
	savedIP= 0
#endif
	LOAD();
	localSP= temporaryPointer - 1;
	temporaryPointer= savedTP;
	internalDrop1Push(NIL_OOP);
	JUMP();
      }
      nonLocalReturn(NIL_OOP);
      NEXT();
    }

    CASE(OpQkRetTop) {
#ifdef TRACE
      printf("OpQkRetTop\n");
#endif
      if (!activated) {
	localIP= savedIP;
#ifdef TRACE
	savedIP= 0
#endif
	LOAD();
	oop result= internalStackTop();
	localSP= temporaryPointer - 1;
	temporaryPointer= savedTP;
	internalDrop1Push(result);
	JUMP();
      }
      nonLocalReturn(internalStackTop());
      NEXT();
    }

#undef nonLocalReturn

#undef specialSend
#undef simplePrim

    } // switch
  } // for
}