more binaryout

[henge/apc.git] / src / binaryout.c

/* Standard */
#include <stdlib.h> //exit, malloc
#include <stdio.h>  //print
#include <stdarg.h> //va_args
#include <string.h> //memset, str*
#include <errno.h>
/* Unicode */
#include <unistd.h>   //u8_* functions
#include <unitypes.h> //uint8_t as a char
#include <unistr.h>   //u32_cpy
#include <unistdio.h> //ulc_fprintf
/* Local */
#include "print.h"
#include "apc.h"
#include "ir.h"
#include "pagenode.h"
#undef  do_error
#define do_error(...) exit(-1)
#define XXH_PRIVATE_API
#include "../xxHash/xxhash.h"
#define STB_IMAGE_IMPLEMENTATION
#include "../stb/stb_image.h"
#define STB_IMAGE_WRITE_IMPLEMENTATION
#include "../stb/stb_image_write.h"

/* Public */
void ir_binout_init(struct ir_class_t*);
 
/* Memory Allocation */
#define  struct_alloc(_T) ((struct _T*) stack_alloc(&datapages, sizeof(struct _T)))
static
struct pagelist_t linkpages, datapages, plinkpages;

enum model_type { SS };
/* Binaryout out structure definitions */
struct bin_img_info_t {
  int height;
  int width;
  int fwidth; //map and frame width
  int fheight; //map and frame height
  int unaligned_width;
  int unaligned_height;
};
struct bin_ht_header_t {
  long start;
  int entries;
  int size;
};
struct bin_default_ht_entry_t {
  uint32_t key; 
  long value;
};
struct bin_var_ht_entry_t {
  uint32_t key;
  long vvalue;
  long mvalue;
};
struct bin_class_header_t {
  struct bin_ht_header_t child_ht;
  struct bin_ht_header_t rootset_ht;
  int namelen;
};
struct bin_set_header_t {
  struct bin_ht_header_t child_ht;
  long sdat_start; //points to setdata_header
  int namelen;
};
struct bin_setdata_header_t {
  struct bin_ht_header_t variant_ht;
  long attach_pos; 
};
struct bin_model_header_t { //one for each framebox, currently
  long facing_array_start;
  enum model_type type;
};
struct bin_frame_header_t {
  int width;
  int height;
  int frames;
  long op_start;
};

struct bin_pixel_t {
  int x, y, z;
  uint32_t ref;
  int attach_idx;
};
struct bin_pixel_node_t {
  struct bin_pixel_node_t* next;
  struct bin_pixel_t data;
};
struct bin_attachment_header_t {
  int num_attachment_lists;
  int num_attachments;
};
struct bin_attachment_t {
  int x, y, z, idx;
  ir_set set;
};
/* Read out of the als, after first ir pass, resolves the
   attach_pos of the src-set that created the attachment_list
   to the header that describes the attachment_list */
struct bin_attachment_list_t {
  int num_attachments;
  struct bin_attachment_t** attachments;
  long filepos;
}; 
struct bin_linklist_t;
struct bin_linklist_t 
{ struct bin_linklist_t* next;
  linkdata               linkdata;
};

struct bin_processed_links_t
{ struct bin_linklist_t* vlink_list;
  int                    vlink_len;
  struct bin_linklist_t* mlink_list;
  int                    mlink_len;
  struct bin_linklist_t* olink_list; //keep track of olink cycles
  int                    olink_len;
  struct bin_linklist_t* dlink_list;
  int                    dlink_len;
};
struct bin_pixel_ht_entry_t
{ uint16_t key;
  uint16_t value;
};

struct bin_pixel_ht_t
{ struct bin_pixel_ht_t* next;
  struct bin_pixel_ht_entry_t[] hash_entries;
};

struct bin_attachment_list_t **attachment_stack, **asp; //attachment_stack, attachment_stack_pointer
FILE* binaryout;


#define  NAMEHASH(name, domain) (XXH32(name, u8_strlen(name), 0XCEED ) & domain) 
#define  REFHASH(ref, domain) (XXH32(&ref, sizeof(uint32_t), 0xCEED) & domain)
static inline
int bin_set_varcount
( ir_set set )
{ int count;
  framebox iter;
  count = 0;
  for (iter = ir_set_framebox(set); iter != NULL; iter = ir_setdata_nextsib(iter))
    count++;
  return count;
}

static inline
int bin_class_sibcount
( ir_class class )
{ int count;
  ir_class iter;
  count = 0;
  for (iter = class; iter != NULL; iter = ir_class_nextsib(iter))
    count++;
  return count;
}

static inline
int bin_set_sibcount
( ir_set set )
{ int count;
  ir_set iter;
  count = 0;
  for (iter = set; iter != NULL; iter = ir_set_nextsib(iter))
    count++;
  return count;
}

/* Given a position and a size, checks if the bytes are null and returns
   the file position to where it started. 1 if not null, 0 if null*/
/* TODO: Determine why fseeking file past end sets bytes to -1, and not 0 */
static inline
int bytes_null( int len, int pos )
{ while(len--)
    { if(fgetc(binaryout) > 0)
        { fseek(binaryout, pos, SEEK_SET);        
          return 1;
        }
    }
  fseek(binaryout, pos, SEEK_SET);
  return 0;
}
/* Checks if the key at entrypos is the same as the parameter key. Returns
   1 if so, 0 if not. */
static inline
int bin_keys_identical
( long entry_pos,
  uint32_t key
)
{ uint32_t curr_key;
  fseek(binaryout, entry_pos, SEEK_SET);
  fscanf(binaryout, "%u", &curr_key);
  if( curr_key == key)
    return 1;
  return 0;     
}


typedef uint32_t RGBA_t;

long bin_traverse_class(ir_class);
/* Takes root class and begins processing */
void
ir_binout_init(ir_class root_class)
{ binaryout = fopen("binaryout", "w+");
  asp = attachment_stack;
  pagelist_init(datapages, (size_t) SYS_PAGESIZE);
  pagelist_init(linkpages, (size_t) SYS_PAGESIZE);
  pagelist_init(plinkpages, (size_t) SYS_PAGESIZE);
  bin_traverse_class(root_class);
}


/* Returns the key position where the hash table entry was inserted. */
#define ENTRY_OCCUPIED() (bytes_null(uint32_t), entry_pos))
#define LOOP_ENTRY(_HTSTART) (entry_pos = _HTSTART)
#define WRITE_DEF_ENTRY() do {                                          \
    if(fseek(binaryout, entry_pos, SEEK_SET) == -1) wprintf("fseek failed with %s", strerror(errno)); \
    fwrite(&def_ht_entry, sizeof def_ht_entry, 1, binaryout);           \
  } while (0)
#define INC_DEF_ENTRY() do { \
    entry_pos += sizeof(def_ht_entry);                                  \
    if(fseek(binaryout, entry_pos, SEEK_SET) == -1) eprintf("fseek failed with %s", strerror(errno)); \
  } while (0)
#define HT_END(_HTEND) (entry_pos >= _HTEND) //just in case at last entry
long
bin_insert_default_ht_entry
( long ht_start,
  int ht_size,
  struct bin_def_ht_entry_t* def_ht_entry,
  int overwrite
)
{ long entry_pos, ht_end;

  ht_end = ht_start + ht_size; 
  entry_pos = ht_start + sizeof(ht_entry) * ht_entry->key;
  fseek(binaryout, entry_pos, SEEK_SET);
  
  if (!ENTRY_OCCUPIED())
    { uprintf("key not occupied\n");
      WRITE_DEF_ENTRY();
    }
  while( ENTRY_OCCUPIED() )
    { if(overwrite)
        { if(bin_keys_identical(entry_pos, ht_entry->key))
            break;      
          else 
           { eprintf("error in hashtable insertion, keys are identical");
           }
        }
      if (HT_END(ht_end))
        LOOP_ENTRY(ht_start);
      else 
        INC_DEF_ENTRY();
    }
  WRITE_DEF_ENTRY();

  return entry_pos;
    
}
long
#define WRITE_VAR_ENTRY() do { \
   if(fseek(binaryout, entry_pos, SEEK_SET) == -1) wprintf("fseek failed with %s", strerror(errno)); \
   fwrite(&var_ht_entry, sizeof var_ht_entry, 1, binaryout);    \
} while (0)
#define INC_VAR_ENTRY() do { \
    entry_pos += sizeof(var_ht_entry);                                  \
    if(fseek(binaryout, entry_pos, SEEK_SET) == -1) eprintf("fseek failed with %s", strerror(errno)); \
  } while (0)
bin_insert_var_ht_entry
( long ht_start,
  int ht_size,
  struct bin_var_ht_entry_t* var_ht_entry,
  int overwrite
)
{ long entry_pos, ht_end;

  ht_end = ht_start + ht_size; 
  entry_pos = ht_start + sizeof(var_ht_entry) * ht_entry->key;
  fseek(binaryout, entry_pos, SEEK_SET);
  
  if (!ENTRY_OCCUPIED())
    { uprintf("key not occupied\n");
      WRITE_VAR_ENTRY();
    }
  while( ENTRY_OCCUPIED() )
    { if(overwrite)
        { if(bin_keys_identical(entry_pos, ht_entry->key))
            break;      
          else 
           { eprintf("error in hashtable insertion, keys are identical");
           }
        }
      if (HT_END(ht_end))
        LOOP_ENTRY(ht_start);
      else 
        INC_VAR_ENTRY();
    }
  WRITE_VAR_ENTRY();

  return entry_pos;

/** @see http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 */
static inline
int bin_calculate_ht_entries
( int entries )
{ entries = (entries << 1) - 1;
  entries |= entries >> 1;
  entries |= entries >> 2;
  entries |= entries >> 4;
  entries |= entries >> 8;
  entries |= entries >> 16;
  return ++entries;
}

/* |     class header   |
   |--------------------|
   |     rootset ht     |
   |--------------------|
   |    rootsets data   |
   |--------------------|
   |    classchild ht   |
   |--------------------|
   |  classchild header |
   |        ...         |
*/

long bin_traverse_set(ir_set);
long
#define DEF_HT_INIT(_START,  _SIZE, _INIT_ENTRIES) do {         \
  _SIZE = _ENTRIES * sizeof(var_ht_entry); \
  _START = ftell(binaryout); \
  fseek(binaryout, _SIZE, SEEK_CUR); \
} while (0)
#define VAR_HT_INIT(_START, _SIZE, _ENTRIES) do { \
  _SIZE = _ENTRIES * sizeof(var_ht_entry);        \
  _START = ftell(binaryout); \
  fseek(binaryout, _SIZE, SEEK_CUR); \
} while (0)
bin_traverse_class
( ir_class class)
{ ir_class citer;
  ir_set siter;
  struct bin_class_header_t class_header;
  struct bin_def_ht_entry_t ht_entry;
  long class_start, classht_start, classht_size, rootsetht_start, rootsetht_size;
  int num_class_entries, num_rootset_entries;
  uint8_t* class_name;

  class_start = ftell(binaryout);
  class_name = ir_class_name(class);

  num_class_entries = bin_calculate_ht_entries(bin_class_sibcount(class));
  num_rootset_entries = bin_calculate_ht_entries(ir_class_rootset(class));

  /* alloc space (before hash tables) for class header */
  class_header.namelen = u8_strlen(class_name);
  fseek(binaryout, class_start + sizeof(class_header) + class_header.namelen ,SEEK_SET);
  
  DEF_HT_INIT(classht_start, classht_size, num_class_entries);
  DEF_HT_INIT(rootsetht_start, rootsetht_size, num_rootset_entries);
  
  /* TODO: Figure out generic way to output headers */
  /* populate class header */
  class_header.child_ht.entries = num_class_entries;
  class_header.child_ht.start = classht_start;
  class_header.child_ht.size = classht_size;
  class_header.rootset_ht.entries = num_rootset_entries;
  class_header.rootset_ht.start = rootsetht_start;
  class_header.rootset_ht.size = rootsetht_size;
  fseek(binaryout, class_start, SEEK_SET); //seek back to where we allocated header
  fwrite(&class_header, sizeof(class_header), 1, binaryout);
  fwrite(class_name, class_header.namelen, 1, binaryout);
 
  /* Start populating root_set hash table */
  for ( siter = ir_class_rootset(class); siter != NULL; siter = ir_set_nextsib(siter))
    { fseek(binaryout, 0, SEEK_END);
      ht_entry.key = NAMEHASH(ir_set_name(siter), num_rootset_entries);
      ht_entry.value = bin_traverse_set(siter);
      bin_insert_def_ht_entry(rootsetht_start, rootsetht_size, &ht_entry, 0);      
    }

  /* Start populating class child hash table */
  for ( citer = ir_class_nextchild(class); citer != NULL; citer = ir_class_nextsib(citer))
    { if(chdir((char*) class_name))
        eprintf("CHDIR %U from %s\n",(char*) class_name,getcwd(NULL,255));
      fseek(binaryout, 0, SEEK_END);
      ht_entry.key = NAMEHASH(ir_class_name(citer), num_class_entries);     
      ht_entry.value = bin_traverse_class(citer);
      bin_insert_def_ht_entry(classht_start, classht_size, &ht_entry, 0);
      if (chdir(".."))
        eprintf("CHDIR ..\n");
    }

  return class_start;
}

long bin_process_sdat( ir_set);

/* |    set header   |--|
   |-----------------|  |
   |     set data    |<-|
   |-----------------|  |
---|   setchild ht   |<--
|  |-----------------|
|->| setchild header |
|->|      ...        |
*/

long
bin_traverse_set
( ir_set set )
{ ir_set iter;
  struct bin_set_header_t header;
  struct bin_def_ht_entry_t ht_entry;
  int num_entries, setname_len;
  long childht_start, childht_size, set_start;
  uint8_t* set_name;

  set_start = ftell(binaryout);
  set_name = ir_set_name(set);
  
  /* alloc space for set header */
  setname_len = u8_strlen(set_name);
  fseek(binaryout, sizeof(struct bin_set_header_t) + setname_len , SEEK_CUR);

  /* process the set data */
  header.sdat_start = bin_process_sdat(set);

  /* Setup child hash table for current sets children */
  num_entries = bin_calculate_ht_entries(bin_set_sibcount(ir_set_nextchild(set)));
  
  DEF_HT_INIT(childht_start, childht_size, num_child);

  /* populate header, write to file */
  header.child_ht.entries = num_entries;
  header.child_ht.start = childht_start;
  header.child_ht.size = childht_size;
  fseek(binaryout, set_start, SEEK_SET);
  fwrite(&header, sizeof(struct bin_set_header_t), 1, binaryout);
  fwrite(set_name, setname_len, 1, binaryout);

  for(iter = ir_set_nextchild(set); iter != NULL; iter = ir_set_nextsib(iter))
    { fseek(binaryout, 0, SEEK_END);
      ht_entry.key = NAMEHASH(ir_set_name(iter), num_entries);
      ht_entry.value = bin_traverse_set(iter);
      bin_insert_def_ht_entry(childht_start, childht_size, &ht_entry, 0);      
    }
  

  ir_set_assign_fpos(set, set_start);
  return set_start;
  
}
/* |   sdat header    |
   |------------------|
   |   num dlinks     |
   |------------------|
   |    dlink len     |
   |------------------|
   |   dlink string   |
   |------------------|
   |    variant ht    |
   |------------------|
   | 1st variant data | -- variant == framebox
   |------------------|
   | 2nd variant data |
   |------------------|
   |       etc.       |
 
*/

static inline void bin_insert_links(struct bin_processed_links_t*, struct bin_ht_header_t*, long);
static inline struct bin_pixel_node_t* bin_find_default_pixel_list(ir_set);
static inline void bin_process_frameboxes(ir_set, struct bin_ht_header_t*, struct bin_pixel_node_t*);
static inline struct bin_processed_links_t* bin_process_links(ir_set, struct bin_processed_links_t*);
static inline void bin_process_dlinks(struct bin_processed_links_t*);

/* Init the variant hash table for the set, process the sets links and add them to link_stack
   and variant hash table, and then output the actual framedata */
long
bin_process_sdat
( ir_set set )
{ struct bin_setdata_header_t header;
  struct bin_attachment_list_t attachment_list;
  struct bin_pixel_node_t *default_pixel_list;
  struct bin_ht_header_t ht_header;
  struct bin_processed_links_t* processed_links_root;
  long varht_start, varht_size, sdat_start;
  int num_entries, num_links;

  sdat_start = ftell(binaryout);
 
  /* Alloc position for sdat_header */
  fseek(binaryout, sizeof(struct bin_setdata_header_t), SEEK_CUR);

  /* set up root for processed_links */
  processed_links_root = stack_alloc(&plinkpages, sizeof(struct bin_processed_links_t));
  processed_links_root->mlink_len = processed_links_root->vlink_len = processed_links_root->dlink_len = processed_links_root->olinks_len = 0;
  processed_links_root = bin_process_links(set, processed_links_root);

  num_links = processed_links_root->mlink_len + processed_links_root->vlink_len;
  
  num_entries = bin_calculate_ht_entries(bin_set_varcount(set) + num_links);
  
  VAR_HT_INIT(varht_start, varht_size, num_entries);

  /* Populate the sdat_header */
  fseek(binaryout, 0, sdat_start);
  header.variant_ht.start =  varht_start;
  header.variant_ht.entries = num_entries;
  header.variant_ht.size = varht_size;
  attachment_list.filepos = header.attach_pos = ftell(binaryout) + sizeof(varht_start) + sizeof(num_entries);
  fwrite(&header, sizeof(header), 1, binaryout);
  fseek(binaryout, 0, SEEK_ENDhttps://en.wikipedia.org/wiki/Generic_programming);

  /* Process dlinks */
  bin_process_dlinks(processed_links_root);
  
  /* Determine the default pixel list for all of the frameboxes */
  default_pixel_list = bin_find_default_pixel_list(set);
  /* Output each framebox, and insert it into the variant hash table */
  bin_process_frameboxes(set, &ht_header, default_pixel_list);

  /* insert the links that were processed into the variant hash table */
  bin_insert_links(processed_links_root, &ht_header, attachment_list.filepos);

  /* TODO: Convert the default pixel list to an attachment_list and then push the  */
  /*       sdats attachment_list onto the attachment_stack so it can be procesed   */
  



  return sdat_start;
}
static inline
void bin_process_dlinks
( struct bin_processed_links_t* processed_links )
{ struct bin_linklist_t* dlink_iter;
  for( dlink_iter = processed_links->dlink_list; dlink_iter != NULL; dlink_iter = dlink_iter->next)
    { /* TODO: Construct its fully qualified name based on its linkdata*/
      
      /* Output an int for its length, and then output the name */
           
          
    }
}

static inline
struct bin_linklist_t* bin_linklist_head
( struct bin_linklist_t* root )
{ struct bin_linklist_t* head;
  head = root;
  while(head->next)
    head = head->next;
  return head;
}

/* We dont know src_pos at this point because this is still in the control flow
   of bin_process_links, which determines the number of links, which determines
   the hash table. */
void
#define PUSH_LINK(_LINK) (*(linkdata*) stack_alloc(&linkpages, sizeof(linkdata)) = _LINK)
bin_process_vlink
( linkdata vlink,
  struct bin_processed_links_t* processed_links)
{ struct bin_linklist_t* llp;
  struct bin_linklist_t* vlink_list_head;
  linkdata new_vlink;
  ir_setdata fiter;
  ir_set trg_set;
  uint8_t* link_name;

  vlink_list_head = bin_linklist_head(processed_links->vlink_list);
  link_name = ir_setdata_name(vlink); 
  if (link_name) 
    { llp =  stack_alloc(&plinkpages, sizeof(bin_linklist_t));
      llp->linkdata = vlink;
      vlink_list_head->next = llp;
      processed_links->vlink_len++;
    }
  else // linking a variant hash table
    { trg_set = ir_linkdata_set(vlink);
      for (fiter = ir_set_framebox(trg_set); fiter != NULL; fiter = ir_setdata_nextsib(fiter))
        { llp = stack_alloc(&plinkpages, sizeof(bin_linklist_t));
          new_vlink = stack_alloc(&plinkpages, sizeof(linkdata));
          ir_data_assign_path(new_vlink,ir_setdata_name(fiter));
          ir_linkdata_assign_set(new_vlink,trg_set);
          ir_linkdata_assign_type(new_vlink,VLINK);
          llp->linkdata = vlink;
          vlink_list_head->next = llp;
          processed_links->vlink_len++;
        }
    }


}

/* Adds an mlink to the stack_alloc, to be processed later */
/* TODO: redo */
void
bin_process_mlink
( linkdata mlink,
  struct bin_processed_links_t* processed_links
)
{ uint8_t* mlink_name;
  struct bin_linklist_t* llp;
  struct bin_linklist_t* mlink_list_head;
  linkdata new_mlink;
  ir_setdata fiter;
  ir_set trg_set;
  

  mlink_list_head = bin_linklist_head(processed_links->mlink_list);
  mlink_name = ir_setdata_name(mlink);

  if(mlink_name) 
    { llp =  stack_alloc(&plinkpages, sizeof(bin_linklist_t));
      llp->linkdata = mlink;
      mlink_list_head->next = llp;
      processed_links->mlink_len++;
    }
  else
    { trg_set = ir_linkdata_set(mlink);
      for (fiter = ir_set_framebox(trg_set); fiter != NULL; fiter = ir_setdata_nextsib(fiter))
        { //TODO: check here if illegal mlink(linking to a opsheet of a vdat not in src_set domain)? 
          llp = stack_alloc(&plinkpages, sizeof(bin_linklist_t));
          new_mlink = stack_alloc(&plinkpages, sizeof(linkdata));
          ir_data_assign_path(new_mlink,ir_setdata_name(fiter));//TODO: assign name
          ir_linkdata_assign_set(new_vlink,trg_set);
          ir_linkdata_assign_type(new_vlink,VLINK);
          llp->linkdata = vlink;
          vlink_list_head->next = llp;
          processed_links->vlink_len++;
        }
    }

  

  return processed_links;
}

/* Determine if olink is already part of the olink_list. 
   if it is, theres a cycle, return 1. Else return 0. */
static inline
int olink_cycle
( linkdata olink,
  struct bin_processed_links_t* processed_links
)
{ struct bin_linklist_t* iter;
  ir_set olink_set;
  olink_set = ir_linkdata_set(olink);
  for( iter = processed_links->olink_list; iter != NULL; iter = iter->next)
    if(ir_linkdata_set(iter->linkdata) == olink_set)
     return 1;  
  
  return 0;
}
/* if olink, process target sets frameboxes(turn into vlinks) and its attachment_list (turn into mlink),
   else its a dlink so just add it to the processed_links list*/
static inline
void bin_process_olink
( ir_set trg_set,
  linkdata olink,
  struct bin_processed_links_t* processed_links_root 
)
{ struct bin_linklist_t* link_list_head;
  struct bin_linklist_t* new_link;

  new_link = stack_alloc(&plinkpages, sizeof(bin_linklist_t));
  if(trg_set) //add olink to list so we can check for cycles
    { bin_set_frameboxes_vlinks(trg_set, processed_links_root); //TODO: 
      bin_set_attachmentlist_mlink(trg_set, processed_links_root); //TODO: 
      link_list_head = bin_linklist_head(processed_links_root->olink_list);
      new_link->linkdata = olink;
      link_list_head->next = new_link;
    }
  else // olink is actually a dynamic link
    { link_list_head = bin_linklist_head(processed_links_root->dlink_list);
      new_link->linkdata = olink;
      link_list_head->next = new_link;
    }
  
}


/* Given a set, determine the number of links it has and process each link and 
   then add them to stack_alloc, where they will be popped off and further processed. */
struct bin_processed_links_t* bin_process_links
( ir_set src_set,
  struct bin_processed_links_t* processed_links_root
)
{ linkdata linkdata;
  ir_set trg_set;
  for(linkdata = ir_set_link(src_set); linkdata != NULL; linkdata = ir_setdata_nextsib((ir_setdata) linkdata))
    { switch (ir_linkdata_type(linkdata)) {
      case OLINK:
        if (olink_cycle(linkdata, processed_links_root)) 
          return processed_links_root; //TODO: what return value? 
        trg_set = ir_linkdata_set(linkdata);
        bin_process_olink(trg_set, linkdata, processed_links_root);
        bin_process_links(trg_set, processed_links_root);       
        break;
      case VLINK:
        bin_process_vlink(linkdata, processed_links_root);
        break;
      case MLINK:
        bin_process_mlink(linkdata, processed_links_root);  
        break;
      case ALINK: //TODO: ?
        break;
      }      
    }
  return processed_links_root;
}

/* Insert both mlinks and vlinks into the link stack, after determining their src_pos. Vlinks 
   have an additional requirement of being added into the variant hash table */
#define FRAME_POS() (entry_pos + sizeof(long))
#define MAP_POS()   (entry_pos + sizeof(long)*2)
#define PUSH_PLINK(_LINK) (*(linkdata*) stack_alloc(&linkpages, sizeof (_LINK)) = _LINK )
void
bin_insert_links
( struct bin_processed_links_t* links,
  struct bin_ht_header_t* ht,
  long attach_pos
)
{ struct bin_plink_t* plp;
  struct bin_var_ht_entry_t ht_entry;
  struct bin_linklist_t* vlinkiter, *mlinkiter;
  long entry_pos;
  
  
  /* Insert vlinks and mlinks into hash table, put v/mlinks on link stack to be processed later */
  for ( vlinkiter = links->vlink_list; vlinkiter != NULL; vlinkiter = vlinkiter->next)
    { ht_entry.key = NAMEHASH(ir_setdata_name(vlinkiter->linkdata), ht->entries);
      ht_entry.fvalue = 0;
      entry_pos = bin_insert_var_ht_entry(ht->start, ht->size, &ht_entry, 0);     
      ir_setdata_assign_fpos(vlinkiter->linkdata, FRAME_POS());
      PUSH_PLINK(vlinkiter->linkdata);
    }
  /* TODO: If name exists in src_set, overwrite. if it dont, print a warning */
  for ( mlinkiter = links->mlink_list; mlinkiter != NULL; mlinkiter = mlinkiter->next)
    { ht_entry.key = NAMEHASH(ir_setdata_name(mlinkiter->linkdata), ht->size);
      ht_entry.mvalue = 0;
      entrypos = bin_insert_var_ht_entry(ht->start, ht->size, &ht_entry, 0);
      ir_setdata_assign_fpos(mlinkiter->linkdata, MAP_POS());
      PUSH_PLINK(mlinkiter->linkdata);
    }
  /* free all the processed links */
  pagelist_free(plinkpages);

     
}

long bin_process_facing(ir_setdata, apc_facing, struct bin_pixel_node_t*);
/* |-------------------|
   |  framebox header  |
   |-------------------|
   | SFACE framesheet  | 
   |-------------------|
   | SWFACE framesheet |
   |-------------------|
   |       etc.        |
*/   
long
bin_process_framebox
( ir_set set,
  ir_setdata framebox,
  struct bin_pixel_node_t* default_pixel_list
)
{ struct bin_model_header_t header;
  long framebox_start, index_pos;
  int i;

  framebox_start = ftell(binaryout);

  /* insert model header */
  header.type = SS;
  header.facing_array_start = framebox_start + sizeof(header);
  fwrite(&header, sizeof(header), 1, binaryout);

  /* Create the index array for framesheet of each direction */
  for ( i = SFACE; i < FACING_MAX; i++)
    { fseek(binaryout, 0, SEEK_END);
      index_pos = bin_process_facing(framebox, i, default_pixel_list); //TODO: finish process_direction
      fseek(binaryout, header.facing_array_start + i * sizeof(long), SEEK_SET);
      fwrite(&index_pos, sizeof(long), 1, binaryout);    
    }
  
  return framebox_start;
}
static inline
struct bin_pixel_ht_t* bin_pixel_ht_alloc
()
{ struct bin_pixel_ht_t* htp;
  if(!(htp = (struct bin_pixel_ht_t*) malloc(sizeof(bin_pixel_ht_t) + )))
     eprintf("error mallocing pixel_ht\n");

       return htp;
}
int bin_insert_pixel_ht_entry
( struct bin_pixel_ht_t* ht,
  struct bin_pixel_ht_entry_t* ht_entry
)
{ 

}

void
bin_process_frameboxes
( ir_set set,
  struct bin_ht_header_t* ht,
  struct bin_pixel_node_t* default_pixel_list
)
{ struct bin_ht_entry_t ht_entry;
  struct bin_pixel_ht_t* ht;
  struct bin_pixel_node_t* pixeliter;
  ir_setdata fiter;

  /* create the default ht */
  ht = bin_ht_alloc();
  for(pixeliter = default_pixel_list; pixeliter != NULL; pixeliter = pixeliter->next)
    { ht_entry.val = 1;
      ht_entry.key = pixel_iter->data.ref;
      bin_insert_pixel_ht_entry(&ht, ;
    }
  


  
  /* Insert variants into hash table to overwrite olink insertions*/
  for ( fiter = ir_set_framebox(set); fiter != NULL; fiter = ir_setdata_nextsib(fiter))
    { fseek(binaryout, 0, SEEK_END);
      ht_entry.key = NAMEHASH(ir_setdata_name(fiter), ht->entries);
        /* create the copy, pass the copy */
      ht_entry.value = bin_process_framebox(set, fiter, default_pixel_list);
      bin_insert_var_ht_entry(ht->start, ht->entries * sizeof(ht_entry), &ht_entry, 1);     
    }

}
/* Determine clipping based on image height/width and frame height/width */
static inline
void bin_set_img_info
( struct bin_img_info_t* img_info,
  ir_setdata frame_data
)
{ ir_frameinfo frameinfo;
  frameinfo = ir_framedata_frameinfo(frame_data);
  img_info->fwidth = frameinfo->w;
  img_info->fheight = frameinfo->h;
  img_info->unaligned_height = img_info->height % img_info->fheight;
  img_info->unaligned_width = img_info->width % img_info->fwidth;

}
/* |-----------------------------|
   |        frame header         |
   |-----------------------------|
   |  pixel data for frame1 - n  |
   |-----------------------------|
   |  op data for frame1 - n     |
   |-----------------------------| */

long
//TODO: THIS SHOULD THE SET SPEC, NOT THE FRAMEBOX NAME
#define GENERATE_FILENAME(_N) ((char*) u8_strcat(_N, png_suffix))
bin_process_facing
( ir_setdata framebox,
  apc_facing facing,
  struct bin_pixel_node_t* default_pixel_list
)
{ struct bin_img_info_t mapsheet_info, framesheet_info;
  int num_mapchannels, num_framechannels, x;
  struct bin_frame_header_t header;
  long facing_start;
  RGBA_t* mapdata, * framedata;
  uint8_t* png_suffix = ".png";
  struct bin_pixel_node_t* map_pixel_list;

  facing_start = ftell(binaryout);

  
  /* Set up data pointers to mapsheet and framesheet, as well as their image infos */
  mapdata = (RGBA_t*) stbi_load(GENERATE_FILENAME(ir_setdata_name((ir_setdata) ir_framebox_mapsheet(framebox,SFACE))), &mapsheet_info.width, &mapsheet_info.width, &num_framechannels , 0);
  framedata = (RGBA_t*) stbi_load(GENERATE_FILENAME(ir_setdata_name((ir_setdata) ir_framebox_framesheet(framebox,SFACE))), &framesheet_info.width, &framesheet_info.height, &num_mapchannels, 0);
  bin_set_img_info(&framesheet_info, ir_framebox_framesheet(framebox, SFACE));
  bin_set_img_info(&mapsheet_info, ir_framebox_mapsheet(framebox, SFACE));

  /* Allocate space for header */
  fseek(binaryout, sizeof(header), SEEK_CUR);
  
  
  /* Output framesheet */
  if(!stbi_write_png(binaryout, framesheet_info.width, framesheet_info.height, 4, mapdata, framesheet_info.fwidth))
     eprintf("error writing out framesheet\n");
  
  /* Output framesheet header */
  header.width = framesheet_info.fwidth;
  header.height = framesheet_info.fheight;
  header.frames = framesheet_info.width / framesheet_info.fwidth; //TODO: division is bad
  header.op_start = ftell(binaryout);
  fseek(binaryout, facing_start, SEEK_SET);
  fwrite(&header, sizeof(header), 1, binaryout);
  fseek(binaryout, 0, SEEK_END);

  

 
  /* Assuming that fheight = image height */
  /* For each mapframe in mapsheet */
  for ( x = 0; x < header.frames; x++)
    { map_pixel_list = bin_mapframe_to_pixel_list(mapsheet_info, 0, x * mapsheet_info.fwidth, mapdata);
      if(!bin_process_map_pixel_list(default_pixel_list, map_pixel_list))
         eprintf("error processing map pixel list\n");
      bin_output_pixel_list(map_pixel_list);
      mapdata = mapsheet_info.fwidth * x; //do we do this in mapframe to pixellist?

    }
 
  
 
  return facing_start;
  
}

/* pixel_list == ops, output up to fwidth amount of them */
void
bin_output_pixel_list(struct bin_pixel_node_t* map_pixel_list);

/* TODO: Please rename all the functions jhc*/
static inline
void bin_number_pixel_list
( struct bin_pixel_node_t* pixel_list )
{ int num = 0;
  struct bin_pixel_node_t* iter;
  while (iter)
    { iter->data.attach_idx = num++;
      iter = iter->next;
    }
}

/* Assuming at this point that map_pixel_list is valid */
static inline
int bin_set_map_pixel_list_attach_idxs
( struct bin_pixel_node_t* default_pixel_list,
  struct bin_pixel_node_t* map_pixel_list
)
{ struct bin_pixel_node_t* mapiter, defaultiter;
  mapiter = map_pixel_list;
  defaultiter = default_pixel_list;
  while (mapiter && defaultiter)
    { /* if mapiter.data.ref == defaultiter.data.ref, assign mapiter index_idx to defaultiter */
      if (mapiter.data.ref == defauliter.data.ref)
        { defaultiter.data.attach_idx = mapiter.data.attach_idx;
          mapiter = mapiter->next;
          defaultiter = defaultiter->next;
        }
      else
        defaultiter = defaultiter->next;
    }
}
int bin_ref_in_pixel_list
( struct bin_pixel_node_t* pixel_list,
  uint32_t ref
)
{ struct bin_pixel_node_t* iter;
  for(iter = pixel_list; iter != NULL; iter = iter->next)
    { if(ref == iter.data.ref)
        return 1;
    }
  return 0;
}

struct bin_pixel_ht_t*
#define PIXEL_HT_SIZE() (sizeof(bin_pixel_ht_entry_t) * SYS_PAGESIZE + sizeof(bin_pixel_ht_t*))
bin_pixel_ht_alloc
( void )
{ struct bin_pixel_ht_t* ht;

  if(!(ht = (struct bin_pixel_ht_t*) malloc( PIXEL_HT_SIZE())))
   eprintf("Memory allocation error in bin_pixel_ht_alloc\n");
  memset(ht, 0, PIXEL_HT_SIZE());

  return ht;
}

void
bin_insert_pixel_ht_entry
( struct bin_pixel_ht_t** ht,
  struct bin_pixel_ht_entry_t* ht_entry
)
{ if(*ht == NULL)
    *ht = bin_pixel_ht_alloc();
    
}

/* Determines if the multiset map_pixel is a subset of the multiset default pixel list.
   0 if invalid, 1 if valid */
static inline
int bin_valid_map_pixel_list
( struct bin_pixel_ht_t* default_ht,
  struct bin_pixel_node_t* map_pixel_list
)
{ struct bin_pixel_node_t* mapiter, *defaultiter, *tmpdefault, tmpmap;
  int i;
  defaultiter = default_pixel_list;
  mapiter = map_pixel_list;

  while(mapiter != NULL)
    { /* hash the ref*/
      /* compare against the default ht */
      /* decrement the value of the found ht_entry */
      /* if(value == 0) */
      /*  return 0 */


    }
  
 
  return 1;
    
}
static inline 
int bin_process_map_pixel_list
( struct bin_pixel_node_t* default_pixel_list,
  struct bin_pixel_node_t* map_pixel_list
)
{ /* Determine if pixel_list is valid */
  if(!bin_valid_map_pixel_list(default_pixel_list, map_pixel_list))
    return 0;
                               
  /* Determine attach_idx of each pixel, as compared to default pixel list */
  
  
}

void
 bin_assign_pixel_idxs
( struct bin_pixel_node_t* pixel_list )
{ 
}

/* Insert pixel(s) into the list, z sorted */
/* number the pixels as you insert them */
struct bin_pixel_node_t*
bin_insert_node_into_list
( struct bin_pixel_node_t** pixel_list_root,
  struct bin_pixel_node_t* pixel_node
)
{ struct bin_pixel_node_t** head_node;

  head_node = pixel_list_root;

  while(*head_node != NULL && head_node->data.ref < pixel_node->data.ref)
    head_node = &((*head_node)->next);

  pixel_node->next = *head_node;
  *head_node = pixel_node;

  return pixel_list_root;


}

/* Returns the non null pixels of a single mapframe (unaligned or not) 
   given any  height and width */
struct bin_pixel_node_t*
bin_mapframe_to_pixel_list
( struct bin_img_info_t* img_info,
  int init_height,
  int init_width,
  RBGA_t* data
)
{ int j, i, fheight, fwidth, fhsize, fwsize;
  RGBA_t* p;
  struct bin_pixel_node_t* pixel_list,* pixel_node;

  pixel_list = NULL;

  /* if frame clips, process unclippign frames */
  
  fwsize = img_info->unaligned_width ? img_info->unaligned_width : img_info->fwidth;
  fhsize = img_info->unaligned_height ? img_info->unaligned_height : img_info->fwidth;


  fwidth = img_info->fwidth;
  fheight = img_info->fheight;
    
  
  /* Process the map*/
  for (i = init_height; i < fhsize; i++)
    { for ( j = init_width; j < fwsize; j++ )
        { if (p = data[i*img_info->width+j])
          { pixel_node = struct_alloc(bin_pixel_node_t);
            /* get ref from 4 bytes of data */  
            pixel_node->data.ref =  (*p) >> 8;
            /* bitshift by ? to get Z */
            pixel_node->data.z = (*p & 0xFF);
            /* set x and y */
            pixel_node->data.x = j;
            pixel_node->data.y = i;
            pixel_node->data.num = 0; 
            pixel_list = bin_insert_node_into_list(&pixel_list, pixel_node);
          }
      }
    }
  
  return pixel_list;
}

static inline
int bin_pixel_list_len
( struct bin_pixel_node_t* pl )
{ struct bin_pixel_node_t* plp;
  int count;
  count = 0;
  plp = pl;
  while(plp)
    { count++;
      plp = plp->next;
    }    
  return count;
}
/* TODO: what are the qualifications for the default pixel list? len and __? */
struct bin_pixel_node_t* 
bin_cmp_default_pixel_lists
( struct bin_pixel_node_t* pl1,
  struct bin_pixel_node_t* pl2
)
{ struct bin_pixel_node_t* pl1p, * pl2p; 
  int i, pl1_len, pl2_len;

  pl1p = pl1;
  pl2p = pl2;
  pl1_len = bin_pixel_list_len(pl1);
  pl2_len = bin_pixel_list_len(pl2);
  
  if (pl1_len > pl2_len)
    return pl1;
  else if (pl1_len < pl2_len)
    return pl2;
  /* pl1 == pl2, make sure that all refs are the same */
  /* TODO: what type of warning/error handling should occur here? */
  for (i = 0; i < pl1_len; i++)
    { if (pl1p->data.ref != pl2p->data.ref)
        eprintf("Error in determining default pixel list\n");
      pl1p = pl1p->next;
      pl2p = pl2p->next;
    }
  return pl1; //doesnt matter which one you return
}

/* Find default framebox, based on the framebox with the most attachments*/
/* Search through first frame of S of each framebox */
struct bin_pixel_node_t*
bin_find_default_pixel_list
( ir_set set)
{ ir_setdata fiter;
  struct bin_pixel_node_t* default_pixel_list, * curr_pixel_list;
  RGBA_t* data;
  int num_channels;
  struct bin_img_info_t img_info;
  
  for (fiter = ir_set_framebox(set); fiter != NULL; fiter = ir_setdata_nextsib(fiter))
    { /* TODO: Stringify the frame name with .png? */
      /* TODO: Add directory changing */
      data = (RGBA_t*) stbi_load(ir_setdata_name((ir_setdata) ir_framebox_mapsheet(fiter,SFACE) ), &img_info.width, &img_info.width, &num_channels, 0);
      bin_set_img_info(&img_info, ir_framebox_mapsheet(fiter, SFACE));
      curr_pixel_list = bin_mapframe_to_pixel_list(&img_info, 0, 0, data);
      default_pixel_list = bin_cmp_default_pixel_lists(curr_pixel_list, default_pixel_list);

      free(data);        
    }

  return default_pixel_list;
}