loading ELF file in C in user space

I am trying to load an ELF file compiled with "gcc -m32 test.c -o test.exe" on Linux in a 64 bit x86 environment. I am trying to load that 32bit file (test.exe) inside a user space ELF loader which has the following core logic (32bit ELF).

The problem is that calling into the returned start address results in a segmentation fault core dump. Here is the code:

void *image_load (char *elf_start, unsigned int size)
{
    Elf32_Ehdr      *hdr    = NULL;
    Elf32_Phdr      *phdr   = NULL;
    unsigned char   *start  = NULL;
    Elf32_Addr      taddr   = 0;
    Elf32_Addr      offset  = 0;
    int i = 0;
    unsigned char *exec = NULL;
    Elf32_Addr      estart = 0;

    hdr = (Elf32_Ehdr *) elf_start;

    if(!is_image_valid(hdr)) {
        printk("image_load:: invalid ELF image\n");
        return 0;
    }

    exec = (unsigned char *)mmap(NULL, size, PROT_READ | PROT_WRITE | PROT_EXEC,
                      MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);

    if(!exec) {
        printk("image_load:: error allocating memory\n");
        return 0;
    }

    // Start with clean memory.
    memset(exec,0x0,size);

    phdr = (Elf32_Phdr *)(elf_start + hdr->e_phoff);

    for(i=0; i < hdr->e_phnum; ++i) {
            if(phdr[i].p_type != PT_LOAD) {
                    continue;
            }
            if(phdr[i].p_filesz > phdr[i].p_memsz) {
                    printk("image_load:: p_filesz > p_memsz\n");
                    munmap(exec, size);
                    return 0;
            }
            if(!phdr[i].p_filesz) {
                    continue;
            }

            // p_filesz can be smaller than p_memsz,
            // the difference is zeroe'd out.
            start = (unsigned char *) (elf_start + phdr[i].p_offset);
            // taddr = phdr[i].p_vaddr + (Elf32_Addr)exec;
            if(!estart) {
                estart = phdr[i].p_paddr;
            }
            taddr = (Elf32_Addr)ex