loop-aes/current/kernel-2.4.36.diff

Before this patch can be applied to kernel, drivers/block/loop.c and
include/linux/loop.h source files must be removed:

    rm -f drivers/block/loop.c include/linux/loop.h

diff -urN linux-2.4.27-noloop/Documentation/Configure.help linux-2.4.27-AES/Documentation/Configure.help
--- linux-2.4.27-noloop/Documentation/Configure.help    Sun Aug  8 14:15:08 2004
+++ linux-2.4.27-AES/Documentation/Configure.help       Sun Aug 15 16:39:41 2004
@@ -620,6 +620,21 @@
 
   If unsure, say N.
 
+AES encrypted loop device support
+CONFIG_BLK_DEV_LOOP_AES
+  If you want to use AES encryption algorithm to encrypt loop devices,
+  say Y here. If you don't know what to do here, say N.
+
+loop encryption key scrubbing support
+CONFIG_BLK_DEV_LOOP_KEYSCRUB
+  Loop encryption key scrubbing moves and inverts key bits in
+  kernel RAM so that the thin oxide which forms the storage
+  capacitor dielectric of DRAM cells is not permitted to develop
+  detectable property. For more info, see Peter Gutmann's paper:
+  http://www.cs.auckland.ac.nz/~pgut001/pubs/secure_del.html
+
+  Paranoid tinfoil hat crowd say Y here, everyone else say N.
+
 ATA/IDE/MFM/RLL support
 CONFIG_IDE
   If you say Y here, your kernel will be able to manage low cost mass
diff -urN linux-2.4.27-noloop/drivers/block/Config.in linux-2.4.27-AES/drivers/block/Config.in
--- linux-2.4.27-noloop/drivers/block/Config.in Sun Aug  8 14:15:13 2004
+++ linux-2.4.27-AES/drivers/block/Config.in    Sun Aug 15 16:39:41 2004
@@ -42,6 +42,10 @@
 dep_tristate 'Promise SATA SX8 support' CONFIG_BLK_DEV_SX8 $CONFIG_PCI
 
 tristate 'Loopback device support' CONFIG_BLK_DEV_LOOP
+if [ "$CONFIG_BLK_DEV_LOOP" != "n" ]; then
+   bool '  AES encrypted loop device support' CONFIG_BLK_DEV_LOOP_AES
+   bool '  loop encryption key scrubbing support' CONFIG_BLK_DEV_LOOP_KEYSCRUB
+fi
 dep_tristate 'Network block device support' CONFIG_BLK_DEV_NBD $CONFIG_NET
 
 tristate 'RAM disk support' CONFIG_BLK_DEV_RAM
diff -urN linux-2.4.27-noloop/drivers/block/loop.c linux-2.4.27-AES/drivers/block/loop.c
--- linux-2.4.27-noloop/drivers/block/loop.c    Thu Jan  1 02:00:00 1970
+++ linux-2.4.27-AES/drivers/block/loop.c       Sun Aug 15 16:39:41 2004
@@ -0,0 +1,2018 @@
+/*
+ *  linux/drivers/block/loop.c
+ *
+ *  Written by Theodore Ts'o, 3/29/93
+ *
+ * Copyright 1993 by Theodore Ts'o.  Redistribution of this file is
+ * permitted under the GNU General Public License.
+ *
+ * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
+ * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
+ *
+ * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
+ * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
+ *
+ * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997
+ *
+ * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998
+ *
+ * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
+ *
+ * Loadable modules and other fixes by AK, 1998
+ *
+ * Make real block number available to downstream transfer functions, enables
+ * CBC (and relatives) mode encryption requiring unique IVs per data block.
+ * Reed H. Petty, rhp@draper.net
+ *
+ * Maximum number of loop devices now dynamic via max_loop module parameter.
+ * Russell Kroll <rkroll@exploits.org> 19990701
+ *
+ * Maximum number of loop devices when compiled-in now selectable by passing
+ * max_loop=<1-255> to the kernel on boot.
+ * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
+ *
+ * Completely rewrite request handling to be make_request_fn style and
+ * non blocking, pushing work to a helper thread. Lots of fixes from
+ * Al Viro too.
+ * Jens Axboe <axboe@suse.de>, Nov 2000
+ *
+ * Support up to 256 loop devices
+ * Heinz Mauelshagen <mge@sistina.com>, Feb 2002
+ *
+ * AES transfer added. IV is now passed as (512 byte) sector number.
+ * Jari Ruusu, May 18 2001
+ *
+ * External encryption module locking bug fixed.
+ * Ingo Rohloff <rohloff@in.tum.de>, June 21 2001
+ *
+ * Make device backed loop work with swap (pre-allocated buffers + queue rewrite).
+ * Jari Ruusu, September 2 2001
+ *
+ * File backed code now uses file->f_op->read/write. Based on Andrew Morton's idea.
+ * Jari Ruusu, May 23 2002
+ *
+ * Backported struct loop_info64 ioctls from 2.6 kernels (64 bit offsets and
+ * 64 bit sizelimits). Added support for removing offset from IV computations.
+ * Jari Ruusu, September 21 2003
+ *
+ * Added support for MD5 IV computation and multi-key operation.
+ * Jari Ruusu, October 8 2003
+ *
+ *
+ * Still To Fix:
+ * - Advisory locking is ignored here.
+ * - Should use an own CAP_* category instead of CAP_SYS_ADMIN
+ */
+
+#include <linux/config.h>
+#include <linux/module.h>
+
+#include <linux/sched.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/stat.h>
+#include <linux/errno.h>
+#include <linux/major.h>
+#include <linux/wait.h>
+#include <linux/blk.h>
+#include <linux/blkpg.h>
+#include <linux/init.h>
+#include <linux/devfs_fs_kernel.h>
+#include <linux/smp_lock.h>
+#include <linux/swap.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include <asm/uaccess.h>
+#include <asm/byteorder.h>
+
+#include <linux/loop.h>                
+#include "../misc/aes.h"
+#include "../misc/md5.h"
+
+#define MAJOR_NR LOOP_MAJOR
+
+static int max_loop = 8;
+static struct loop_device *loop_dev;
+static int *loop_sizes;
+static int *loop_blksizes;
+static int *loop_hardsizes;
+static devfs_handle_t devfs_handle;      /*  For the directory */
+
+#if defined(__x86_64__) && defined(CONFIG_IA32_EMULATION)
+# include <asm/ioctl32.h>
+# define IOCTL32_COMPATIBLE_PTR ((void*)sys_ioctl)
+#endif
+#if (defined(__sparc__) || defined(__sparc64__)) && defined(CONFIG_SPARC32_COMPAT)
+  extern int register_ioctl32_conversion(unsigned int cmd, int (*handler)(unsigned int, unsigned int, unsigned long, struct file *));
+  extern int unregister_ioctl32_conversion(unsigned int cmd);
+  extern int sys_ioctl(unsigned int fd, unsigned int cmd, unsigned long arg);
+# define IOCTL32_COMPATIBLE_PTR ((void*)sys_ioctl)
+#endif
+
+/*
+ * Transfer functions
+ */
+static int transfer_none(struct loop_device *lo, int cmd, char *raw_buf,
+                        char *loop_buf, int size, int real_block)
+{
+       /* this code is only called from file backed loop  */
+       /* and that code expects this function to be no-op */
+
+       if (current->need_resched)
+               {set_current_state(TASK_RUNNING);schedule();}
+       return 0;
+}
+
+static int transfer_xor(struct loop_device *lo, int cmd, char *raw_buf,
+                       char *loop_buf, int size, int real_block)
+{
+       char    *in, *out, *key;
+       int     i, keysize;
+
+       if (cmd == READ) {
+               in = raw_buf;
+               out = loop_buf;
+       } else {
+               in = loop_buf;
+               out = raw_buf;
+       }
+
+       key = lo->lo_encrypt_key;
+       keysize = lo->lo_encrypt_key_size;
+       for (i = 0; i < size; i++)
+               *out++ = *in++ ^ key[(i & 511) % keysize];
+       if (current->need_resched)
+               {set_current_state(TASK_RUNNING);schedule();}
+       return 0;
+}
+
+static int none_status(struct loop_device *lo, struct loop_info *info)
+{
+       return 0;
+}
+
+static int xor_status(struct loop_device *lo, struct loop_info *info)
+{
+       if (info->lo_encrypt_key_size <= 0)
+               return -EINVAL;
+       return 0;
+}
+
+struct loop_func_table none_funcs = {
+       number: LO_CRYPT_NONE,
+       transfer: transfer_none,
+       init: none_status,
+};     
+
+struct loop_func_table xor_funcs = {
+       number: LO_CRYPT_XOR,
+       transfer: transfer_xor,
+       init: xor_status,
+};     
+
+#if CONFIG_BLK_DEV_LOOP_AES
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+# define KEY_ALLOC_COUNT  128
+#else
+# define KEY_ALLOC_COUNT  64
+#endif
+
+typedef struct {
+    aes_context *keyPtr[KEY_ALLOC_COUNT];
+    unsigned    keyMask;
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+    rwlock_t    rwlock;
+    unsigned    reversed;
+    unsigned    blocked;
+    struct timer_list timer;
+#endif
+} AESmultiKey;
+
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+static void keyScrubWork(AESmultiKey *m)
+{
+    aes_context *a0, *a1;
+    u_int32_t *p;
+    int x, y, z;
+
+    z = m->keyMask + 1;
+    for(x = 0; x < z; x++) {
+        a0 = m->keyPtr[x];
+        a1 = m->keyPtr[x + z];
+        memcpy(a1, a0, sizeof(aes_context));
+        m->keyPtr[x] = a1;
+        m->keyPtr[x + z] = a0;
+        p = (u_int32_t *) a0;
+        y = sizeof(aes_context) / sizeof(u_int32_t);
+        while(y > 0) {
+            *p ^= 0xFFFFFFFF;
+            p++;
+            y--;
+        }
+    }
+    m->reversed ^= 1;
+
+    /* try to flush dirty cache data to RAM */
+#if defined(CONFIG_X86_64) || (defined(CONFIG_X86) && !defined(CONFIG_M386) && !defined(CONFIG_CPU_386))
+    __asm__ __volatile__ ("wbinvd": : :"memory");
+#else
+    mb();
+#endif
+}
+
+/* called only from loop thread process context */
+static void keyScrubThreadFn(AESmultiKey *m)
+{
+    write_lock(&m->rwlock);
+    if(!m->blocked) keyScrubWork(m);
+    write_unlock(&m->rwlock);
+}
+
+static void keyScrubTimerInit(struct loop_device *lo)
+{
+    AESmultiKey     *m;
+    unsigned long   expire;
+    static void keyScrubTimerFn(unsigned long);
+
+    m = (AESmultiKey *)lo->key_data;
+    expire = jiffies + HZ;
+    init_timer(&m->timer);
+    m->timer.expires = expire;
+    m->timer.data = (unsigned long)lo;
+    m->timer.function = keyScrubTimerFn;
+    add_timer(&m->timer);
+}
+
+/* called only from timer handler context */
+static void keyScrubTimerFn(unsigned long d)
+{
+    struct loop_device *lo = (struct loop_device *)d;
+    extern void loop_add_keyscrub_fn(struct loop_device *, void (*)(void *), void *);
+
+    /* rw lock needs process context, so make loop thread do scrubbing */
+    loop_add_keyscrub_fn(lo, (void (*)(void*))keyScrubThreadFn, lo->key_data);
+    /* start timer again */
+    keyScrubTimerInit(lo);
+}
+#endif
+
+static AESmultiKey *allocMultiKey(void)
+{
+    AESmultiKey *m;
+    aes_context *a;
+    int x = 0, n;
+
+    m = (AESmultiKey *) kmalloc(sizeof(AESmultiKey), GFP_KERNEL);
+    if(!m) return 0;
+    memset(m, 0, sizeof(AESmultiKey));
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+    rwlock_init(&m->rwlock);
+    init_timer(&m->timer);
+    again:
+#endif
+
+    n = PAGE_SIZE / sizeof(aes_context);
+    if(!n) n = 1;
+
+    a = (aes_context *) kmalloc(sizeof(aes_context) * n, GFP_KERNEL);
+    if(!a) {
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+        if(x) kfree(m->keyPtr[0]);
+#endif
+        kfree(m);
+        return 0;
+    }
+
+    while((x < KEY_ALLOC_COUNT) && n) {
+        m->keyPtr[x] = a;
+        a++;
+        x++;
+        n--;
+    }
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+    if(x < 2) goto again;
+#endif
+    return m;
+}
+
+static void clearAndFreeMultiKey(AESmultiKey *m)
+{
+    aes_context *a;
+    int x, n;
+
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+    /* stop scrub timer. loop thread was killed earlier */
+    del_timer_sync(&m->timer);
+    /* make sure allocated keys are in original order */
+    if(m->reversed) keyScrubWork(m);
+#endif
+    n = PAGE_SIZE / sizeof(aes_context);
+    if(!n) n = 1;
+
+    x = 0;
+    while(x < KEY_ALLOC_COUNT) {
+        a = m->keyPtr[x];
+        if(!a) break;
+        memset(a, 0, sizeof(aes_context) * n);
+        kfree(a);
+        x += n;
+    }
+
+    kfree(m);
+}
+
+static int multiKeySetup(struct loop_device *lo, unsigned char *k)
+{
+    AESmultiKey *m;
+    aes_context *a;
+    int x, y, n, err = 0;
+    union {
+        u_int32_t     w[8]; /* needed for 4 byte alignment for b[] */
+        unsigned char b[32];
+    } un;
+
+    if(lo->lo_key_owner != current->uid && !capable(CAP_SYS_ADMIN))
+        return -EPERM;
+
+    m = (AESmultiKey *)lo->key_data;
+    if(!m) return -ENXIO;
+
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+    /* temporarily prevent loop thread from messing with keys */
+    write_lock(&m->rwlock);
+    m->blocked = 1;
+    /* make sure allocated keys are in original order */
+    if(m->reversed) keyScrubWork(m);
+    write_unlock(&m->rwlock);
+#endif
+    n = PAGE_SIZE / sizeof(aes_context);
+    if(!n) n = 1;
+
+    x = 0;
+    while(x < KEY_ALLOC_COUNT) {
+        if(!m->keyPtr[x]) {
+            a = (aes_context *) kmalloc(sizeof(aes_context) * n, GFP_KERNEL);
+            if(!a) {
+                err = -ENOMEM;
+                goto error_out;
+            }
+            y = x;
+            while((y < (x + n)) && (y < KEY_ALLOC_COUNT)) {
+                m->keyPtr[y] = a;
+                a++;
+                y++;
+            }
+        }
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+        if(x >= 64) {
+            x++;
+            continue;
+        }
+#endif
+        if(copy_from_user(&un.b[0], k, 32)) {
+            err = -EFAULT;
+            goto error_out;
+        }
+        aes_set_key(m->keyPtr[x], &un.b[0], lo->lo_encrypt_key_size, 0);
+        k += 32;
+        x++;
+    }
+    m->keyMask = 0x3F;          /* range 0...63 */
+    lo->lo_flags |= 0x100000;   /* multi-key (info exported to user space) */
+    memset(&un.b[0], 0, 32);
+error_out:
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+    /* re-enable loop thread key scrubbing */
+    write_lock(&m->rwlock);
+    m->blocked = 0;
+    write_unlock(&m->rwlock);
+#endif
+    return err;
+}
+
+void loop_compute_sector_iv(int devSect, u_int32_t *ivout)
+{
+    ivout[0] = cpu_to_le32(devSect);
+    ivout[3] = ivout[2] = ivout[1] = 0;
+}
+
+void loop_compute_md5_iv(int devSect, u_int32_t *ivout, u_int32_t *data)
+{
+    int         x;
+#if defined(__BIG_ENDIAN)
+    int         y, e;
+#endif
+    u_int32_t   buf[16];
+
+    ivout[0] = 0x67452301;
+    ivout[1] = 0xefcdab89;
+    ivout[2] = 0x98badcfe;
+    ivout[3] = 0x10325476;
+
+#if defined(__BIG_ENDIAN)
+    y = 7;
+    e = 16;
+    do {
+        if (!y) {
+            e = 12;
+            /* md5_transform_CPUbyteorder wants data in CPU byte order */
+            /* devSect is already in CPU byte order -- no need to convert */
+            /* 32 bits of sector number + 24 zero bits */
+            buf[12] = devSect;
+            buf[13] = 0x80000000;
+            /* 4024 bits == 31 * 128 bit plaintext blocks + 56 bits of sector number */
+            buf[14] = 4024;
+            buf[15] = 0;
+        }
+        x = 0;
+        do {
+            buf[x    ] = cpu_to_le32(data[0]);
+            buf[x + 1] = cpu_to_le32(data[1]);
+            buf[x + 2] = cpu_to_le32(data[2]);
+            buf[x + 3] = cpu_to_le32(data[3]);
+            x += 4;
+            data += 4;
+        } while (x < e);
+        md5_transform_CPUbyteorder(&ivout[0], &buf[0]);
+    } while (--y >= 0);
+    ivout[0] = cpu_to_le32(ivout[0]);
+    ivout[1] = cpu_to_le32(ivout[1]);
+    ivout[2] = cpu_to_le32(ivout[2]);
+    ivout[3] = cpu_to_le32(ivout[3]);
+#else
+    x = 6;
+    do {
+        md5_transform_CPUbyteorder(&ivout[0], data);
+        data += 16;
+    } while (--x >= 0);
+    memcpy(buf, data, 48);
+    /* md5_transform_CPUbyteorder wants data in CPU byte order */
+    /* devSect is already in CPU byte order -- no need to convert */
+    /* 32 bits of sector number + 24 zero bits */
+    buf[12] = devSect;
+    buf[13] = 0x80000000;
+    /* 4024 bits == 31 * 128 bit plaintext blocks + 56 bits of sector number */
+    buf[14] = 4024;
+    buf[15] = 0;
+    md5_transform_CPUbyteorder(&ivout[0], &buf[0]);
+#endif
+}
+
+static int transfer_aes(struct loop_device *lo, int cmd, char *raw_buf,
+          char *loop_buf, int size, int devSect)
+{
+    aes_context     *a;
+    AESmultiKey     *m;
+    int             x;
+    unsigned        y;
+    u_int32_t       iv[8];
+
+    if(!size || (size & 511)) {
+        return -EINVAL;
+    }
+    m = (AESmultiKey *)lo->key_data;
+    y = m->keyMask;
+    if(cmd == READ) {
+        while(size) {
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+            read_lock(&m->rwlock);
+#endif
+            a = m->keyPtr[((unsigned)devSect) & y];
+            if(y) {
+                memcpy(&iv[0], raw_buf, 16);
+                raw_buf += 16;
+                loop_buf += 16;
+            } else {
+                loop_compute_sector_iv(devSect, &iv[0]);
+            }
+            x = 15;
+            do {
+                memcpy(&iv[4], raw_buf, 16);
+                aes_decrypt(a, raw_buf, loop_buf);
+                *((u_int32_t *)(&loop_buf[ 0])) ^= iv[0];
+                *((u_int32_t *)(&loop_buf[ 4])) ^= iv[1];
+                *((u_int32_t *)(&loop_buf[ 8])) ^= iv[2];
+                *((u_int32_t *)(&loop_buf[12])) ^= iv[3];
+                if(y && !x) {
+                    raw_buf -= 496;
+                    loop_buf -= 496;
+                    loop_compute_md5_iv(devSect, &iv[4], (u_int32_t *)(&loop_buf[16]));
+                } else {
+                    raw_buf += 16;
+                    loop_buf += 16;
+                    memcpy(&iv[0], raw_buf, 16);
+                }
+                aes_decrypt(a, raw_buf, loop_buf);
+                *((u_int32_t *)(&loop_buf[ 0])) ^= iv[4];
+                *((u_int32_t *)(&loop_buf[ 4])) ^= iv[5];
+                *((u_int32_t *)(&loop_buf[ 8])) ^= iv[6];
+                *((u_int32_t *)(&loop_buf[12])) ^= iv[7];
+                if(y && !x) {
+                    raw_buf += 512;
+                    loop_buf += 512;
+                } else {
+                    raw_buf += 16;
+                    loop_buf += 16;
+                }
+            } while(--x >= 0);
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+            read_unlock(&m->rwlock);
+#endif
+            if(current->need_resched) {set_current_state(TASK_RUNNING);schedule();}
+            size -= 512;
+            devSect++;
+        }
+    } else {
+        while(size) {
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+            read_lock(&m->rwlock);
+#endif
+            a = m->keyPtr[((unsigned)devSect) & y];
+            if(y) {
+                /* on 2.4 and later kernels, real raw_buf is not doing */
+                /* any writes now so it can be used as temp buffer */
+                memcpy(raw_buf, loop_buf, 512);
+                loop_compute_md5_iv(devSect, &iv[0], (u_int32_t *)(&raw_buf[16]));
+                x = 15;
+                do {
+                    iv[0] ^= *((u_int32_t *)(&raw_buf[ 0]));
+                    iv[1] ^= *((u_int32_t *)(&raw_buf[ 4]));
+                    iv[2] ^= *((u_int32_t *)(&raw_buf[ 8]));
+                    iv[3] ^= *((u_int32_t *)(&raw_buf[12]));
+                    aes_encrypt(a, (unsigned char *)(&iv[0]), raw_buf);
+                    memcpy(&iv[0], raw_buf, 16);
+                    raw_buf += 16;
+                    iv[0] ^= *((u_int32_t *)(&raw_buf[ 0]));
+                    iv[1] ^= *((u_int32_t *)(&raw_buf[ 4]));
+                    iv[2] ^= *((u_int32_t *)(&raw_buf[ 8]));
+                    iv[3] ^= *((u_int32_t *)(&raw_buf[12]));
+                    aes_encrypt(a, (unsigned char *)(&iv[0]), raw_buf);
+                    memcpy(&iv[0], raw_buf, 16);
+                    raw_buf += 16;
+                } while(--x >= 0);
+                loop_buf += 512;
+            } else {
+                loop_compute_sector_iv(devSect, &iv[0]);
+                x = 15;
+                do {
+                    iv[0] ^= *((u_int32_t *)(&loop_buf[ 0]));
+                    iv[1] ^= *((u_int32_t *)(&loop_buf[ 4]));
+                    iv[2] ^= *((u_int32_t *)(&loop_buf[ 8]));
+                    iv[3] ^= *((u_int32_t *)(&loop_buf[12]));
+                    aes_encrypt(a, (unsigned char *)(&iv[0]), raw_buf);
+                    memcpy(&iv[0], raw_buf, 16);
+                    loop_buf += 16;
+                    raw_buf += 16;
+                    iv[0] ^= *((u_int32_t *)(&loop_buf[ 0]));
+                    iv[1] ^= *((u_int32_t *)(&loop_buf[ 4]));
+                    iv[2] ^= *((u_int32_t *)(&loop_buf[ 8]));
+                    iv[3] ^= *((u_int32_t *)(&loop_buf[12]));
+                    aes_encrypt(a, (unsigned char *)(&iv[0]), raw_buf);
+                    memcpy(&iv[0], raw_buf, 16);
+                    loop_buf += 16;
+                    raw_buf += 16;
+                } while(--x >= 0);
+            }
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+            read_unlock(&m->rwlock);
+#endif
+            if(current->need_resched) {set_current_state(TASK_RUNNING);schedule();}
+            size -= 512;
+            devSect++;
+        }
+    }
+    return(0);
+}
+
+static int keySetup_aes(struct loop_device *lo, struct loop_info *info)
+{
+    AESmultiKey     *m;
+    union {
+        u_int32_t     w[8]; /* needed for 4 byte alignment for b[] */
+        unsigned char b[32];
+    } un;
+
+    lo->key_data = m = allocMultiKey();
+    if(!m) return(-ENOMEM);
+    memcpy(&un.b[0], &info->lo_encrypt_key[0], 32);
+    aes_set_key(m->keyPtr[0], &un.b[0], info->lo_encrypt_key_size, 0);
+    memset(&info->lo_encrypt_key[0], 0, sizeof(info->lo_encrypt_key));
+    memset(&un.b[0], 0, 32);
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+    keyScrubTimerInit(lo);
+#endif
+    return(0);
+}
+
+static int keyClean_aes(struct loop_device *lo)
+{
+    if(lo->key_data) {
+        clearAndFreeMultiKey((AESmultiKey *)lo->key_data);
+        lo->key_data = 0;
+    }
+    return(0);
+}
+
+static int handleIoctl_aes(struct loop_device *lo, int cmd, unsigned long arg)
+{
+    int err;
+
+    switch (cmd) {
+    case LOOP_MULTI_KEY_SETUP:
+        err = multiKeySetup(lo, (unsigned char *)arg);
+        break;
+    default:
+        err = -EINVAL;
+    }
+    return err;
+}
+
+static struct loop_func_table funcs_aes = {
+    number:     16,     /* 16 == AES */
+    transfer:   transfer_aes,
+    init:       keySetup_aes,
+    release:    keyClean_aes,
+    ioctl:      handleIoctl_aes
+};
+
+EXPORT_SYMBOL(loop_compute_sector_iv);
+EXPORT_SYMBOL(loop_compute_md5_iv);
+#endif /* CONFIG_BLK_DEV_LOOP_AES */
+
+/* xfer_funcs[0] is special - its release function is never called */
+struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
+       &none_funcs,
+       &xor_funcs,
+#if CONFIG_BLK_DEV_LOOP_AES
+       [LO_CRYPT_AES] = &funcs_aes,
+#endif
+};
+
+/*
+ *  First number of 'lo_prealloc' is the default number of RAM pages
+ *  to pre-allocate for each device backed loop. Every (configured)
+ *  device backed loop pre-allocates this amount of RAM pages unless
+ *  later 'lo_prealloc' numbers provide an override. 'lo_prealloc'
+ *  overrides are defined in pairs: loop_index,number_of_pages
+ */
+static int lo_prealloc[9] = { 125, 999, 0, 999, 0, 999, 0, 999, 0 };
+#define LO_PREALLOC_MIN 4    /* minimum user defined pre-allocated RAM pages */
+#define LO_PREALLOC_MAX 512  /* maximum user defined pre-allocated RAM pages */
+
+#ifdef MODULE
+MODULE_PARM(lo_prealloc, "1-9i");
+MODULE_PARM_DESC(lo_prealloc, "Number of pre-allocated pages [,index,pages]...");
+#else
+static int __init lo_prealloc_setup(char *str)
+{
+       int x, y, z;
+
+       for (x = 0; x < (sizeof(lo_prealloc) / sizeof(int)); x++) {
+               z = get_option(&str, &y);
+               if (z > 0)
+                       lo_prealloc[x] = y;
+               if (z < 2)
+                       break;
+       }
+       return 1;
+}
+__setup("lo_prealloc=", lo_prealloc_setup);
+#endif
+
+/*
+ * This is loop helper thread nice value in range
+ * from 0 (low priority) to -20 (high priority).
+ */
+#if defined(DEF_NICE) && defined(DEF_COUNTER)
+static int lo_nice = -20;   /* old scheduler default */
+#else
+static int lo_nice = -1;    /* O(1) scheduler default */
+#endif
+
+#ifdef MODULE
+MODULE_PARM(lo_nice, "1i");
+MODULE_PARM_DESC(lo_nice, "Loop thread scheduler nice (0 ... -20)");
+#else
+static int __init lo_nice_setup(char *str)
+{
+       int y;
+
+       if (get_option(&str, &y) == 1)
+               lo_nice = y;
+       return 1;
+}
+__setup("lo_nice=", lo_nice_setup);
+#endif
+
+typedef struct {
+       struct buffer_head      **q0;
+       struct buffer_head      **q1;
+       struct buffer_head      **q2;
+       int                     x0;
+       int                     x1;
+       int                     x2;
+} que_look_up_table;
+
+static void loop_prealloc_cleanup(struct loop_device *lo)
+{
+       struct buffer_head *bh;
+
+       while ((bh = lo->lo_bh_free)) {
+               __free_page(bh->b_page);
+               lo->lo_bh_free = bh->b_reqnext;
+               bh->b_reqnext = NULL;
+               kmem_cache_free(bh_cachep, bh);
+       }
+}
+
+static int loop_prealloc_init(struct loop_device *lo, int y)
+{
+       struct buffer_head *bh;
+       int x;
+
+       if(!y) {
+               y = lo_prealloc[0];
+               for (x = 1; x < (sizeof(lo_prealloc) / sizeof(int)); x += 2) {
+                       if (lo_prealloc[x + 1] && (lo->lo_number == lo_prealloc[x])) {
+                               y = lo_prealloc[x + 1];
+                               break;
+                       }
+               }
+       }
+       lo->lo_bh_flsh = (y * 3) / 4;
+
+       for (x = 0; x < y; x++) {
+               bh = kmem_cache_alloc(bh_cachep, SLAB_KERNEL);
+               if (!bh) {
+                       loop_prealloc_cleanup(lo);
+                       return 1;
+               }
+               bh->b_page = alloc_page(GFP_KERNEL);
+               if (!bh->b_page) {
+                       bh->b_reqnext = NULL;
+                       kmem_cache_free(bh_cachep, bh);
+                       loop_prealloc_cleanup(lo);
+                       return 1;
+               }
+               bh->b_reqnext = lo->lo_bh_free;
+               lo->lo_bh_free = bh;
+       }
+       return 0;
+}
+
+static void loop_add_queue_last(struct loop_device *lo, struct buffer_head *bh, struct buffer_head **q)
+{
+       unsigned long flags;
+
+       spin_lock_irqsave(&lo->lo_lock, flags);
+       if (*q) {
+               bh->b_reqnext = (*q)->b_reqnext;
+               (*q)->b_reqnext = bh;
+       } else {
+               bh->b_reqnext = bh;
+       }
+       *q = bh;
+       spin_unlock_irqrestore(&lo->lo_lock, flags);
+
+       if (waitqueue_active(&lo->lo_bh_wait))
+               wake_up_interruptible(&lo->lo_bh_wait);
+}
+
+static void loop_add_queue_first(struct loop_device *lo, struct buffer_head *bh, struct buffer_head **q)
+{
+       spin_lock_irq(&lo->lo_lock);
+       if (*q) {
+               bh->b_reqnext = (*q)->b_reqnext;
+               (*q)->b_reqnext = bh;
+       } else {
+               bh->b_reqnext = bh;
+               *q = bh;
+       }
+       spin_unlock_irq(&lo->lo_lock);
+}
+
+static struct buffer_head *loop_get_bh(struct loop_device *lo, int *list_nr,
+                                       que_look_up_table *qt)
+{
+       struct buffer_head *bh = NULL, *last;
+
+       spin_lock_irq(&lo->lo_lock);
+       if ((last = *qt->q0)) {
+               bh = last->b_reqnext;
+               if (bh == last)
+                       *qt->q0 = NULL;
+               else
+                       last->b_reqnext = bh->b_reqnext;
+               bh->b_reqnext = NULL;
+               *list_nr = qt->x0;
+       } else if ((last = *qt->q1)) {
+               bh = last->b_reqnext;
+               if (bh == last)
+                       *qt->q1 = NULL;
+               else
+                       last->b_reqnext = bh->b_reqnext;
+               bh->b_reqnext = NULL;
+               *list_nr = qt->x1;
+       } else if ((last = *qt->q2)) {
+               bh = last->b_reqnext;
+               if (bh == last)
+                       *qt->q2 = NULL;
+               else
+                       last->b_reqnext = bh->b_reqnext;
+               bh->b_reqnext = NULL;
+               *list_nr = qt->x2;
+       }
+       spin_unlock_irq(&lo->lo_lock);
+       return bh;
+}
+
+static void loop_put_buffer(struct loop_device *lo, struct buffer_head *b)
+{
+       unsigned long flags;
+       int wk;
+
+       spin_lock_irqsave(&lo->lo_lock, flags);
+       b->b_reqnext = lo->lo_bh_free;
+       lo->lo_bh_free = b;
+       wk = lo->lo_bh_need;
+       spin_unlock_irqrestore(&lo->lo_lock, flags);
+
+       if (wk && waitqueue_active(&lo->lo_bh_wait))
+               wake_up_interruptible(&lo->lo_bh_wait);
+}
+
+static void loop_end_io_transfer_wr(struct buffer_head *bh, int uptodate)
+{
+       struct loop_device *lo = &loop_dev[MINOR(bh->b_dev)];
+       struct buffer_head *rbh = bh->b_private;
+
+       rbh->b_reqnext = NULL;
+       rbh->b_end_io(rbh, uptodate);
+       loop_put_buffer(lo, bh);
+       if (atomic_dec_and_test(&lo->lo_pending))
+               wake_up_interruptible(&lo->lo_bh_wait);
+}
+
+static void loop_end_io_transfer_rd(struct buffer_head *bh, int uptodate)
+{
+       struct loop_device *lo = &loop_dev[MINOR(bh->b_dev)];
+
+       if (!uptodate)
+               loop_end_io_transfer_wr(bh, uptodate);
+       else
+               loop_add_queue_last(lo, bh, &lo->lo_bh_que0);
+}
+
+static struct buffer_head *loop_get_buffer(struct loop_device *lo,
+               struct buffer_head *rbh, int from_thread, int rw)
+{
+       struct buffer_head *bh;
+       struct page *p;
+       unsigned long flags;
+
+       spin_lock_irqsave(&lo->lo_lock, flags);
+       bh = lo->lo_bh_free;
+       if (bh) {
+               lo->lo_bh_free = bh->b_reqnext;
+               if (from_thread)
+                       lo->lo_bh_need = 0;
+       } else {
+               if (from_thread)
+                       lo->lo_bh_need = 1;
+       }
+       spin_unlock_irqrestore(&lo->lo_lock, flags);
+       if (!bh)
+               return (struct buffer_head *)0;
+
+       p = bh->b_page;
+       memset(bh, 0, sizeof(struct buffer_head));
+       bh->b_page = p;
+
+       bh->b_private = rbh;
+       bh->b_size = rbh->b_size;
+       bh->b_dev = rbh->b_rdev;
+       bh->b_rdev = lo->lo_device;
+       bh->b_state = (1 << BH_Req) | (1 << BH_Mapped) | (1 << BH_Lock);
+       bh->b_data = page_address(bh->b_page);
+       bh->b_end_io = (rw == WRITE) ? loop_end_io_transfer_wr : loop_end_io_transfer_rd;
+       bh->b_rsector = rbh->b_rsector + lo->lo_offs_sec;
+       init_waitqueue_head(&bh->b_wait);
+
+       return bh;
+}
+
+static int figure_loop_size(struct loop_device *lo)
+{
+       loff_t size, offs;
+       unsigned int x;
+       int err = 0;
+       kdev_t lodev = lo->lo_device;
+
+       offs = lo->lo_offset;
+       if (S_ISREG(lo->lo_backing_file->f_dentry->d_inode->i_mode)) {
+               size = lo->lo_backing_file->f_dentry->d_inode->i_size;
+       } else {
+               offs &= ~((loff_t)511);
+               if (blk_size[MAJOR(lodev)])
+                       size = (loff_t)(blk_size[MAJOR(lodev)][MINOR(lodev)]) << BLOCK_SIZE_BITS;
+               else
+                       size = 1024*1024*1024; /* unknown size */
+       }
+       if ((offs > 0) && (offs < size)) {
+               size -= offs;
+       } else {
+               if (offs)
+                       err = -EINVAL;
+               lo->lo_offset = 0;
+               lo->lo_offs_sec = lo->lo_iv_remove = 0;
+       }
+       if ((lo->lo_sizelimit > 0) && (lo->lo_sizelimit <= size)) {
+               size = lo->lo_sizelimit;
+       } else {
+               if (lo->lo_sizelimit)
+                       err = -EINVAL;
+               lo->lo_sizelimit = 0;
+       }
+       size >>= BLOCK_SIZE_BITS;
+
+       /*
+        * Unfortunately, if we want to do I/O on the device,
+        * the number of 1024-byte blocks has to fit into unsigned int
+        */
+       x = (unsigned int)size;
+       if ((loff_t)x != size) {
+               err = -EFBIG;
+               size = 0;
+       }
+
+       loop_sizes[lo->lo_number] = size;
+       return err;
+}
+
+static int loop_file_io(struct file *file, char *buf, int size, loff_t *ppos, int w)
+{
+       mm_segment_t fs;
+       int x, y, z;
+
+       y = 0;
+       do {
+               z = size - y;
+               fs = get_fs();
+               set_fs(get_ds());
+               if (w) {
+                       x = file->f_op->write(file, buf + y, z, ppos);
+                       set_fs(fs);
+               } else {
+                       x = file->f_op->read(file, buf + y, z, ppos);
+                       set_fs(fs);
+                       if (!x)
+                               return 1;
+               }
+               if (x < 0) {
+                       if ((x == -EAGAIN) || (x == -ENOMEM) || (x == -ERESTART) || (x == -EINTR)) {
+                               run_task_queue(&tq_disk);
+                               set_current_state(TASK_INTERRUPTIBLE);
+                               schedule_timeout(HZ / 2);
+                               continue;
+                       }
+                       return 1;
+               }
+               y += x;
+       } while (y < size);
+       return 0;
+}
+
+static int do_bh_filebacked(struct loop_device *lo, struct buffer_head *bh, int rw)
+{
+       loff_t pos;
+       struct file *file = lo->lo_backing_file;
+       char *data, *buf;
+       unsigned int size, len;
+       unsigned long IV;
+
+       pos = ((loff_t) bh->b_rsector << 9) + lo->lo_offset;
+       buf = page_address(lo->lo_bh_free->b_page);
+       len = bh->b_size;
+       data = bh_kmap(bh);
+       IV = bh->b_rsector;
+       if (!lo->lo_iv_remove)
+               IV += lo->lo_offs_sec;
+       while (len > 0) {
+               if (lo->lo_encrypt_type == LO_CRYPT_NONE) {
+                       /* this code relies that NONE transfer is a no-op */
+                       buf = data;
+               }
+               size = PAGE_SIZE;
+               if (size > len)
+                       size = len;
+               if (rw == WRITE) {
+                       if (lo_do_transfer(lo, WRITE, buf, data, size, IV)) {
+                               printk(KERN_ERR "loop%d: write transfer error, sector %lu\n", lo->lo_number, IV);
+                               goto kunmap_and_out;
+                       }
+                       if (loop_file_io(file, buf, size, &pos, 1)) {
+                               printk(KERN_ERR "loop%d: write i/o error, sector %lu\n", lo->lo_number, IV);
+                               goto kunmap_and_out;
+                       }
+               } else {
+                       if (loop_file_io(file, buf, size, &pos, 0)) {
+                               printk(KERN_ERR "loop%d: read i/o error, sector %lu\n", lo->lo_number, IV);
+                               goto kunmap_and_out;
+                       }
+                       if (lo_do_transfer(lo, READ, buf, data, size, IV)) {
+                               printk(KERN_ERR "loop%d: read transfer error, sector %lu\n", lo->lo_number, IV);
+                               goto kunmap_and_out;
+                       }
+                       flush_dcache_page(bh->b_page);
+               }
+               data += size;
+               len -= size;
+               IV += size >> 9;
+       }
+       bh_kunmap(bh);
+       return 0;
+
+kunmap_and_out:
+       bh_kunmap(bh);
+       return 1;
+}
+
+static int loop_make_request(request_queue_t *q, int rw, struct buffer_head *rbh)
+{
+       struct buffer_head *bh;
+       struct loop_device *lo;
+       char *md;
+
+       set_current_state(TASK_RUNNING);
+       if (!buffer_locked(rbh))
+               BUG();
+
+       if (MINOR(rbh->b_rdev) >= max_loop)
+               goto out;
+
+       lo = &loop_dev[MINOR(rbh->b_rdev)];
+       spin_lock_irq(&lo->lo_lock);
+       if (lo->lo_state != Lo_bound)
+               goto inactive;
+       atomic_inc(&lo->lo_pending);
+       spin_unlock_irq(&lo->lo_lock);
+
+       if (rw == WRITE) {
+               if (lo->lo_flags & LO_FLAGS_READ_ONLY)
+                       goto err;
+       } else if (rw == READA) {
+               rw = READ;
+       } else if (rw != READ) {
+               printk(KERN_ERR "loop%d: unknown command (%d)\n", lo->lo_number, rw);
+               goto err;
+       }
+
+       /*
+        * file backed, queue for loop_thread to handle
+        */
+       if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
+               loop_add_queue_last(lo, rbh, (rw == WRITE) ? &lo->lo_bh_que1 : &lo->lo_bh_que0);
+               return 0;
+       }
+
+       /*
+        * device backed, just remap rdev & rsector for NONE transfer
+        */
+       if (lo->lo_encrypt_type == LO_CRYPT_NONE) {
+               rbh->b_rsector += lo->lo_offs_sec;
+               rbh->b_rdev = lo->lo_device;
+               generic_make_request(rw, rbh);
+               if (atomic_dec_and_test(&lo->lo_pending))
+                       wake_up_interruptible(&lo->lo_bh_wait);
+               return 0;
+       }
+
+       /*
+        * device backed, start reads and writes now if buffer available
+        */
+       bh = loop_get_buffer(lo, rbh, 0, rw);
+       if (!bh) {
+               /* just queue request and let thread handle alloc later */
+               loop_add_queue_last(lo, rbh, (rw == WRITE) ? &lo->lo_bh_que1 : &lo->lo_bh_que2);
+               return 0;
+       }
+       if (rw == WRITE) {
+               int trv;
+               md = bh_kmap(rbh);
+               trv = lo_do_transfer(lo, WRITE, bh->b_data, md, bh->b_size, bh->b_rsector - lo->lo_iv_remove);
+               bh_kunmap(rbh);
+               if (trv) {
+                       loop_put_buffer(lo, bh);
+                       goto err;
+               }
+       }
+       generic_make_request(rw, bh);
+       return 0;
+
+err:
+       if (atomic_dec_and_test(&lo->lo_pending))
+               wake_up_interruptible(&lo->lo_bh_wait);
+out:
+       buffer_IO_error(rbh);
+       return 0;
+inactive:
+       spin_unlock_irq(&lo->lo_lock);
+       goto out;
+}
+
+/*
+ * worker thread that handles reads/writes to file backed loop devices,
+ * to avoid blocking in our make_request_fn. it also does loop decrypting
+ * on reads for block backed loop, as that is too heavy to do from
+ * b_end_io context where irqs may be disabled.
+ */
+static int loop_thread(void *data)
+{
+       struct loop_device *lo = data;
+       struct buffer_head *bh, *xbh;
+       int x, rw, qi = 0, flushcnt = 0;
+       wait_queue_t waitq;
+       que_look_up_table qt[4] = {
+               { &lo->lo_bh_que0, &lo->lo_bh_que1, &lo->lo_bh_que2, 0, 1, 2 },
+               { &lo->lo_bh_que2, &lo->lo_bh_que0, &lo->lo_bh_que1, 2, 0, 1 },
+               { &lo->lo_bh_que0, &lo->lo_bh_que2, &lo->lo_bh_que1, 0, 2, 1 },
+               { &lo->lo_bh_que1, &lo->lo_bh_que0, &lo->lo_bh_que2, 1, 0, 2 }
+       };
+       char *md;
+       static const struct rlimit loop_rlim_defaults[RLIM_NLIMITS] = INIT_RLIMITS;
+
+       init_waitqueue_entry(&waitq, current);
+       memcpy(&current->rlim[0], &loop_rlim_defaults[0], sizeof(current->rlim));
+       daemonize();
+       exit_files(current);
+       reparent_to_init();
+
+       sprintf(current->comm, "loop%d", lo->lo_number);
+
+       spin_lock_irq(&current->sigmask_lock);
+       sigfillset(&current->blocked);
+       flush_signals(current);
+       spin_unlock_irq(&current->sigmask_lock);
+
+       if (lo_nice > 0)
+               lo_nice = 0;
+       if (lo_nice < -20)
+               lo_nice = -20;
+#if defined(DEF_NICE) && defined(DEF_COUNTER)
+       /* old scheduler syntax */
+       current->policy = SCHED_OTHER;
+       current->nice = lo_nice;
+#else
+       /* O(1) scheduler syntax */
+       set_user_nice(current, lo_nice);
+#endif
+
+       spin_lock_irq(&lo->lo_lock);
+       lo->lo_state = Lo_bound;
+       atomic_inc(&lo->lo_pending);
+       spin_unlock_irq(&lo->lo_lock);
+
+       current->flags |= PF_NOIO;
+#if defined(PF_NOFREEZE)
+       current->flags |= PF_NOFREEZE;
+#elif defined(PF_IOTHREAD)
+       current->flags |= PF_IOTHREAD;
+#endif
+
+       /*
+        * up sem, we are running
+        */
+       up(&lo->lo_sem);
+
+       for (;;) {
+               add_wait_queue(&lo->lo_bh_wait, &waitq);
+               for (;;) {
+                       set_current_state(TASK_INTERRUPTIBLE);
+                       if (!atomic_read(&lo->lo_pending))
+                               break;
+
+                       x = 0;
+                       spin_lock_irq(&lo->lo_lock);
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+                       if(lo->lo_keyscrub_fn) x = 1;
+#endif
+                       if (lo->lo_bh_que0) {
+                               x = 1;
+                       } else if (lo->lo_bh_que1 || lo->lo_bh_que2) {
+                               /* file backed works too because lo->lo_bh_need == 0 */
+                               if (lo->lo_bh_free || !lo->lo_bh_need)
+                                       x = 1;
+                       }
+                       spin_unlock_irq(&lo->lo_lock);
+                       if (x)
+                               break;
+
+                       schedule();
+               }
+               set_current_state(TASK_RUNNING);
+               remove_wait_queue(&lo->lo_bh_wait, &waitq);
+
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+               if(lo->lo_keyscrub_fn) {
+                       (*lo->lo_keyscrub_fn)(lo->lo_keyscrub_ptr);
+                       lo->lo_keyscrub_fn = 0;
+               }
+#endif
+               /*
+                * could be woken because of tear-down, not because of
+                * pending work
+                */
+               if (!atomic_read(&lo->lo_pending))
+                       break;
+
+               /*
+                * read queues using alternating order to prevent starvation
+                */
+               bh = loop_get_bh(lo, &x, &qt[++qi & 3]);
+               if (!bh)
+                       continue;
+
+               /*
+                *  x  list tag        usage(buffer-allocated)
+                * --- --------------  -----------------------
+                *  0  lo->lo_bh_que0  dev-read(y) / file-read
+                *  1  lo->lo_bh_que1  dev-write(n) / file-write
+                *  2  lo->lo_bh_que2  dev-read(n)
+                */
+               rw = (x == 1) ? WRITE : READ;
+               if ((x >= 1) && !(lo->lo_flags & LO_FLAGS_DO_BMAP)) {
+                       /* loop_make_request didn't allocate a buffer, do that now */
+                       xbh = loop_get_buffer(lo, bh, 1, rw);
+                       if (!xbh) {
+                               run_task_queue(&tq_disk);
+                               flushcnt = 0;
+                               loop_add_queue_first(lo, bh, (rw == WRITE) ? &lo->lo_bh_que1 : &lo->lo_bh_que2);
+                               /* lo->lo_bh_need should be 1 now, go back to sleep */
+                               continue;
+                       }
+                       if (rw == WRITE) {
+                               int trv;
+                               md = bh_kmap(bh);
+                               trv = lo_do_transfer(lo, WRITE, xbh->b_data, md, xbh->b_size, xbh->b_rsector - lo->lo_iv_remove);
+                               bh_kunmap(bh);
+                               if (trv) {
+                                       loop_put_buffer(lo, xbh);
+                                       buffer_IO_error(bh);
+                                       atomic_dec(&lo->lo_pending);
+                                       continue;
+                               }
+                       }
+                       generic_make_request(rw, xbh);
+
+                       /* start I/O if there are no more requests lacking buffers */
+                       x = 0;
+                       spin_lock_irq(&lo->lo_lock);
+                       if (!lo->lo_bh_que1 && !lo->lo_bh_que2)
+                               x = 1;
+                       spin_unlock_irq(&lo->lo_lock);
+                       if (x || (++flushcnt >= lo->lo_bh_flsh)) {
+                               run_task_queue(&tq_disk);
+                               flushcnt = 0;
+                       }
+
+                       /* request not completely processed yet */
+                       continue;
+               }
+               if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
+                       /* request is for file backed device */
+                       x = do_bh_filebacked(lo, bh, rw);
+                       bh->b_reqnext = NULL;
+                       bh->b_end_io(bh, !x);
+               } else {
+                       /* device backed read has completed, do decrypt now */
+                       xbh = bh->b_private;
+                       /* must not use bh->b_rsector as IV, as it may be modified by LVM at this point */
+                       /* instead, recompute IV from original request */
+                       md = bh_kmap(xbh);
+                       x = lo_do_transfer(lo, READ, bh->b_data, md, bh->b_size, xbh->b_rsector + lo->lo_offs_sec - lo->lo_iv_remove);
+                       flush_dcache_page(xbh->b_page);
+                       bh_kunmap(xbh);
+                       xbh->b_reqnext = NULL;
+                       xbh->b_end_io(xbh, !x);
+                       loop_put_buffer(lo, bh);
+               }
+
+               /*
+                * woken both for pending work and tear-down, lo_pending
+                * will hit zero then
+                */
+               if (atomic_dec_and_test(&lo->lo_pending))
+                       break;
+       }
+
+       up(&lo->lo_sem);
+       return 0;
+}
+
+static void loop_set_softblksz(struct loop_device *lo, kdev_t dev)
+{
+       int bs = 0, x;
+
+       if (blksize_size[MAJOR(lo->lo_device)])
+               bs = blksize_size[MAJOR(lo->lo_device)][MINOR(lo->lo_device)];
+       if (!bs)
+               bs = BLOCK_SIZE;
+       if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
+               x = loop_sizes[lo->lo_number];
+               if ((bs == 8192) && (x & 7))
+                       bs = 4096;
+               if ((bs == 4096) && (x & 3))
+                       bs = 2048;
+               if ((bs == 2048) && (x & 1))
+                       bs = 1024;
+       }
+       set_blocksize(dev, bs);
+}
+
+static int loop_set_fd(struct loop_device *lo, struct file *lo_file, kdev_t dev,
+                      unsigned int arg)
+{
+       struct file     *file;
+       struct inode    *inode;
+       kdev_t          lo_device;
+       int             lo_flags = 0, hardsz = 512;
+       int             error;
+
+       MOD_INC_USE_COUNT;
+
+       error = -EBUSY;
+       if (lo->lo_state != Lo_unbound)
+               goto out;
+
+       error = -EBADF;
+       file = fget(arg);
+       if (!file)
+               goto out;
+
+       error = -EINVAL;
+       inode = file->f_dentry->d_inode;
+
+       if (!(file->f_mode & FMODE_WRITE))
+               lo_flags |= LO_FLAGS_READ_ONLY;
+
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+       lo->lo_keyscrub_fn = 0;
+#endif
+       lo->lo_offset = lo->lo_sizelimit = 0;
+       lo->lo_offs_sec = lo->lo_iv_remove = 0;
+       lo->lo_bh_free = lo->lo_bh_que2 = lo->lo_bh_que1 = lo->lo_bh_que0 = NULL;
+       lo->lo_bh_need = lo->lo_bh_flsh = 0;
+       init_waitqueue_head(&lo->lo_bh_wait);
+       if (S_ISBLK(inode->i_mode)) {
+               lo_device = inode->i_rdev;
+               if (lo_device == dev) {
+                       error = -EBUSY;
+                       goto out_putf;
+               }
+               if (loop_prealloc_init(lo, 0)) {
+                       error = -ENOMEM;
+                       goto out_putf;
+               }
+               hardsz = get_hardsect_size(lo_device);
+       } else if (S_ISREG(inode->i_mode)) {
+               /*
+                * If we can't read - sorry. If we only can't write - well,
+                * it's going to be read-only.
+                */
+               if (!file->f_op || !file->f_op->read)
+                       goto out_putf;
+
+               if (!file->f_op->write)
+                       lo_flags |= LO_FLAGS_READ_ONLY;
+
+               lo_device = inode->i_dev;
+               lo_flags |= LO_FLAGS_DO_BMAP;
+               if (loop_prealloc_init(lo, 1)) {
+                       error = -ENOMEM;
+                       goto out_putf;
+               }
+               error = 0;
+       } else
+               goto out_putf;
+
+       get_file(file);
+
+       if ((S_ISREG(inode->i_mode) && IS_RDONLY(inode)) || is_read_only(lo_device)
+           || !(lo_file->f_mode & FMODE_WRITE))
+               lo_flags |= LO_FLAGS_READ_ONLY;
+
+       set_device_ro(dev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);
+
+       lo->lo_device = lo_device;
+       lo->lo_flags = lo_flags;
+       if(lo_flags & LO_FLAGS_READ_ONLY)
+               lo->lo_flags |= 0x200000; /* export to user space */
+       lo->lo_backing_file = file;
+       lo->transfer = NULL;
+       lo->ioctl = NULL;
+       if (figure_loop_size(lo)) {
+               error = -EFBIG;
+               goto out_cleanup;
+       }
+
+       if (lo_flags & LO_FLAGS_DO_BMAP) {
+               lo->old_gfp_mask = inode->i_mapping->gfp_mask;
+               inode->i_mapping->gfp_mask &= ~(__GFP_IO|__GFP_FS);
+               inode->i_mapping->gfp_mask |= __GFP_HIGH;
+       } else {
+               lo->old_gfp_mask = -1;
+       }
+
+       loop_hardsizes[MINOR(dev)] = hardsz;
+       loop_set_softblksz(lo, dev);
+
+       error = kernel_thread(loop_thread, lo, CLONE_FS | CLONE_FILES | CLONE_SIGHAND);
+       if(error < 0)
+               goto out_mapping;
+       down(&lo->lo_sem);
+       fput(file);
+       return 0;
+
+ out_mapping:
+       if(lo->old_gfp_mask != -1)
+               inode->i_mapping->gfp_mask = lo->old_gfp_mask;
+ out_cleanup:
+       loop_prealloc_cleanup(lo);
+       fput(file);
+ out_putf:
+       fput(file);
+ out:
+       MOD_DEC_USE_COUNT;
+       return error;
+}
+
+static int loop_release_xfer(struct loop_device *lo)
+{
+       int err = 0;
+       if (lo->lo_encrypt_type) {
+               struct loop_func_table *xfer= xfer_funcs[lo->lo_encrypt_type];
+               lo->transfer = NULL;
+               if (xfer && xfer->release)
+                       err = xfer->release(lo);
+               if (xfer && xfer->unlock)
+                       xfer->unlock(lo);
+               lo->lo_encrypt_type = 0;
+       }
+       return err;
+}
+
+static int loop_init_xfer(struct loop_device *lo, int type,struct loop_info *i)
+{
+       int err = 0;
+       if (type) {
+               struct loop_func_table *xfer = xfer_funcs[type];
+               if (xfer->init)
+                       err = xfer->init(lo, i);
+               if (!err) {
+                       lo->lo_encrypt_type = type;
+                       if (xfer->lock)
+                               xfer->lock(lo);
+               }
+       }
+       return err;
+}
+
+static int loop_clr_fd(struct loop_device *lo, struct block_device *bdev)
+{
+       struct file *filp = lo->lo_backing_file;
+       int gfp = lo->old_gfp_mask;
+
+       if (lo->lo_state != Lo_bound)
+               return -ENXIO;
+       if (lo->lo_refcnt > 1)  /* we needed one fd for the ioctl */
+               return -EBUSY;
+       if (filp==NULL)
+               return -EINVAL;
+
+       spin_lock_irq(&lo->lo_lock);
+       lo->lo_state = Lo_rundown;
+       if (atomic_dec_and_test(&lo->lo_pending))
+               wake_up_interruptible(&lo->lo_bh_wait);
+       spin_unlock_irq(&lo->lo_lock);
+
+       down(&lo->lo_sem);
+
+       loop_prealloc_cleanup(lo);
+       lo->lo_backing_file = NULL;
+
+       loop_release_xfer(lo);
+       lo->transfer = NULL;
+       lo->ioctl = NULL;
+       lo->lo_device = 0;
+       lo->lo_encrypt_type = 0;
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+       lo->lo_keyscrub_fn = 0;
+#endif
+       lo->lo_offset = lo->lo_sizelimit = 0;
+       lo->lo_offs_sec = lo->lo_iv_remove = 0;
+       lo->lo_encrypt_key_size = 0;
+       lo->lo_flags = 0;
+       memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
+       memset(lo->lo_name, 0, LO_NAME_SIZE);
+       memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
+       loop_sizes[lo->lo_number] = 0;
+       invalidate_bdev(bdev, 0);
+       if (gfp != -1)
+               filp->f_dentry->d_inode->i_mapping->gfp_mask = gfp;
+       lo->lo_state = Lo_unbound;
+       fput(filp);
+       MOD_DEC_USE_COUNT;
+       return 0;
+}
+
+static void
+loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
+{
+       memset(info64, 0, sizeof(*info64));
+       info64->lo_number = info->lo_number;
+       info64->lo_device = info->lo_device;
+       info64->lo_inode = info->lo_inode;
+       info64->lo_rdevice = info->lo_rdevice;
+       info64->lo_offset = info->lo_offset;
+       info64->lo_encrypt_type = info->lo_encrypt_type;
+       info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
+       info64->lo_flags = info->lo_flags;
+       info64->lo_init[0] = info->lo_init[0];
+       info64->lo_init[1] = info->lo_init[1];
+       if (info->lo_encrypt_type == 18) /* LO_CRYPT_CRYPTOAPI */
+               memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
+       else
+               memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
+       memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
+}
+
+static int
+loop_info64_to_old(struct loop_info64 *info64, struct loop_info *info)
+{
+       memset(info, 0, sizeof(*info));
+       info->lo_number = info64->lo_number;
+       info->lo_device = info64->lo_device;
+       info->lo_inode = info64->lo_inode;
+       info->lo_rdevice = info64->lo_rdevice;
+       info->lo_offset = info64->lo_offset;
+       info->lo_encrypt_type = info64->lo_encrypt_type;
+       info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
+       info->lo_flags = info64->lo_flags;
+       info->lo_init[0] = info64->lo_init[0];
+       info->lo_init[1] = info64->lo_init[1];
+       if (info->lo_encrypt_type == 18) /* LO_CRYPT_CRYPTOAPI */
+               memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
+       else
+               memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
+       memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
+
+       /* error in case values were truncated */
+       if (info->lo_device != info64->lo_device ||
+           info->lo_rdevice != info64->lo_rdevice ||
+           info->lo_inode != info64->lo_inode ||
+           info->lo_offset != info64->lo_offset ||
+           info64->lo_sizelimit)
+               return -EOVERFLOW;
+
+       return 0;
+}
+
+static int loop_set_status(struct loop_device *lo, kdev_t dev, struct loop_info64 *info, struct loop_info *oldinfo)
+{
+       int err;
+       unsigned int type;
+
+       if (lo->lo_encrypt_key_size && lo->lo_key_owner != current->uid &&
+           !capable(CAP_SYS_ADMIN))
+               return -EPERM;
+       if (lo->lo_state != Lo_bound)
+               return -ENXIO;
+       if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
+               return -EINVAL;
+       type = info->lo_encrypt_type;
+       if (type >= MAX_LO_CRYPT || xfer_funcs[type] == NULL)
+               return -EINVAL;
+       if (type == LO_CRYPT_XOR && info->lo_encrypt_key_size == 0)
+               return -EINVAL;
+       err = loop_release_xfer(lo);
+       if (err)
+               return err;     
+
+       if ((loff_t)info->lo_offset < 0) {
+               /* negative offset == remove offset from IV computations */
+               lo->lo_offset = -(info->lo_offset);
+               lo->lo_iv_remove = lo->lo_offset >> 9;
+       } else {
+               /* positive offset == include offset in IV computations */
+               lo->lo_offset = info->lo_offset;
+               lo->lo_iv_remove = 0;
+       }
+       lo->lo_offs_sec = lo->lo_offset >> 9;
+       lo->lo_sizelimit = info->lo_sizelimit;
+       err = figure_loop_size(lo);
+       if (err)
+               return err;
+       loop_set_softblksz(lo, dev);
+
+       /* transfer init function for 2.4 kernels takes old style struct */
+       err = loop_init_xfer(lo, type, oldinfo);
+       /* copy key -- just in case transfer init func modified it */
+       memcpy(info->lo_encrypt_key, oldinfo->lo_encrypt_key, sizeof(info->lo_encrypt_key));
+       if (err)
+               return err;
+
+       strncpy(lo->lo_name, info->lo_file_name, LO_NAME_SIZE);
+       strncpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
+       lo->transfer = xfer_funcs[type]->transfer;
+       lo->ioctl = xfer_funcs[type]->ioctl;
+       lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
+       lo->lo_init[0] = info->lo_init[0];
+       lo->lo_init[1] = info->lo_init[1];
+       if (info->lo_encrypt_key_size) {
+               memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
+                      info->lo_encrypt_key_size);
+               lo->lo_key_owner = current->uid;
+       }
+
+       return 0;
+}
+
+static int loop_get_status(struct loop_device *lo, struct loop_info64 *info)
+{
+       struct file *file = lo->lo_backing_file;
+
+       if (lo->lo_state != Lo_bound)
+               return -ENXIO;
+       memset(info, 0, sizeof(*info));
+       info->lo_number = lo->lo_number;
+       info->lo_device = kdev_t_to_nr(file->f_dentry->d_inode->i_dev);
+       info->lo_inode = file->f_dentry->d_inode->i_ino;
+       info->lo_rdevice = kdev_t_to_nr(lo->lo_device);
+       info->lo_offset = lo->lo_iv_remove ? -(lo->lo_offset) : lo->lo_offset;
+       info->lo_sizelimit = lo->lo_sizelimit;
+       info->lo_flags = lo->lo_flags;
+       strncpy(info->lo_file_name, lo->lo_name, LO_NAME_SIZE);
+       strncpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
+       info->lo_encrypt_type = lo->lo_encrypt_type;
+       if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
+               info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
+               memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
+                      lo->lo_encrypt_key_size);
+               info->lo_init[0] = lo->lo_init[0];
+               info->lo_init[1] = lo->lo_init[1];
+       }
+       return 0;
+}
+
+static int
+loop_set_status_n(struct loop_device *lo, kdev_t dev, void *arg, int n)
+{
+       struct loop_info info;
+       struct loop_info64 info64;
+       int err;
+
+       if (n) {
+               if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
+                       return -EFAULT;
+               /* truncation errors can be ignored here as transfer init func only wants key bits */
+               loop_info64_to_old(&info64, &info);
+       } else {
+               if (copy_from_user(&info, arg, sizeof (struct loop_info)))
+                       return -EFAULT;
+               loop_info64_from_old(&info, &info64);
+       }
+       err = loop_set_status(lo, dev, &info64, &info);
+       memset(&info.lo_encrypt_key[0], 0, sizeof(info.lo_encrypt_key));
+       memset(&info64.lo_encrypt_key[0], 0, sizeof(info64.lo_encrypt_key));
+       return err;
+}
+
+static int
+loop_get_status_old(struct loop_device *lo, struct loop_info *arg) {
+       struct loop_info info;
+       struct loop_info64 info64;
+       int err = 0;
+
+       if (!arg)
+               err = -EINVAL;
+       if (!err)
+               err = loop_get_status(lo, &info64);
+       if (!err)
+               err = loop_info64_to_old(&info64, &info);
+       if (!err && copy_to_user(arg, &info, sizeof(info)))
+               err = -EFAULT;
+
+       return err;
+}
+
+static int
+loop_get_status64(struct loop_device *lo, struct loop_info64 *arg) {
+       struct loop_info64 info64;
+       int err = 0;
+
+       if (!arg)
+               err = -EINVAL;
+       if (!err)
+               err = loop_get_status(lo, &info64);
+       if (!err && copy_to_user(arg, &info64, sizeof(info64)))
+               err = -EFAULT;
+
+       return err;
+}
+
+static int lo_ioctl(struct inode * inode, struct file * file,
+       unsigned int cmd, unsigned long arg)
+{
+       struct loop_device *lo;
+       int dev, err;
+
+       if (!inode)
+               return -EINVAL;
+       if (MAJOR(inode->i_rdev) != MAJOR_NR) {
+               printk(KERN_WARNING "lo_ioctl: pseudo-major != %d\n",
+                      MAJOR_NR);
+               return -ENODEV;
+       }
+       dev = MINOR(inode->i_rdev);
+       if (dev >= max_loop)
+               return -ENODEV;
+       lo = &loop_dev[dev];
+       down(&lo->lo_ctl_mutex);
+       switch (cmd) {
+       case LOOP_SET_FD:
+               err = loop_set_fd(lo, file, inode->i_rdev, arg);
+               break;
+       case LOOP_CLR_FD:
+               err = loop_clr_fd(lo, inode->i_bdev);
+               break;
+       case LOOP_SET_STATUS:
+               err = loop_set_status_n(lo, inode->i_rdev, (void *) arg, 0);
+               break;
+       case LOOP_GET_STATUS:
+               err = loop_get_status_old(lo, (struct loop_info *) arg);
+               break;
+       case LOOP_SET_STATUS64:
+               err = loop_set_status_n(lo, inode->i_rdev, (void *) arg, 1);
+               break;
+       case LOOP_GET_STATUS64:
+               err = loop_get_status64(lo, (struct loop_info64 *) arg);
+               break;
+       case BLKGETSIZE:
+               if (lo->lo_state != Lo_bound) {
+                       err = -ENXIO;
+                       break;
+               }
+               err = put_user((unsigned long)loop_sizes[lo->lo_number] << 1, (unsigned long *) arg);
+               break;
+       case BLKGETSIZE64:
+               if (lo->lo_state != Lo_bound) {
+                       err = -ENXIO;
+                       break;
+               }
+               err = put_user((u64)loop_sizes[lo->lo_number] << 10, (u64*)arg);
+               break;
+       case BLKBSZGET:
+       case BLKBSZSET:
+       case BLKSSZGET:
+       case BLKROGET:
+       case BLKROSET:
+               err = blk_ioctl(inode->i_rdev, cmd, arg);
+               break;
+       default:
+               err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
+       }
+       up(&lo->lo_ctl_mutex);
+       return err;
+}
+
+static int lo_open(struct inode *inode, struct file *file)
+{
+       struct loop_device *lo;
+       int     dev;
+
+       if (!inode)
+               return -EINVAL;
+       if (MAJOR(inode->i_rdev) != MAJOR_NR) {
+               printk(KERN_WARNING "lo_open: pseudo-major != %d\n", MAJOR_NR);
+               return -ENODEV;
+       }
+       dev = MINOR(inode->i_rdev);
+       if (dev >= max_loop)
+               return -ENODEV;
+
+       lo = &loop_dev[dev];
+       MOD_INC_USE_COUNT;
+       down(&lo->lo_ctl_mutex);
+       lo->lo_refcnt++;
+       up(&lo->lo_ctl_mutex);
+       return 0;
+}
+
+static int lo_release(struct inode *inode, struct file *file)
+{
+       struct loop_device *lo;
+       int     dev;
+
+       if (!inode)
+               return 0;
+       if (MAJOR(inode->i_rdev) != MAJOR_NR) {
+               printk(KERN_WARNING "lo_release: pseudo-major != %d\n",
+                      MAJOR_NR);
+               return 0;
+       }
+       dev = MINOR(inode->i_rdev);
+       if (dev >= max_loop)
+               return 0;
+
+       lo = &loop_dev[dev];
+       down(&lo->lo_ctl_mutex);
+       --lo->lo_refcnt;
+       up(&lo->lo_ctl_mutex);
+       MOD_DEC_USE_COUNT;
+       return 0;
+}
+
+static struct block_device_operations lo_fops = {
+       owner:          THIS_MODULE,
+       open:           lo_open,
+       release:        lo_release,
+       ioctl:          lo_ioctl,
+};
+
+/*
+ * And now the modules code and kernel interface.
+ */
+MODULE_PARM(max_loop, "i");
+MODULE_PARM_DESC(max_loop, "Maximum number of loop devices (1-256)");
+MODULE_LICENSE("GPL");
+
+int loop_register_transfer(struct loop_func_table *funcs)
+{
+       if ((unsigned)funcs->number >= MAX_LO_CRYPT || xfer_funcs[funcs->number])
+               return -EINVAL;
+       xfer_funcs[funcs->number] = funcs;
+       return 0;
+}
+
+int loop_unregister_transfer(int number)
+{
+       struct loop_device *lo;
+
+       if ((unsigned)number >= MAX_LO_CRYPT)
+               return -EINVAL;
+       for (lo = &loop_dev[0]; lo < &loop_dev[max_loop]; lo++) {
+               int type = lo->lo_encrypt_type;
+               if (type == number) {
+                       loop_release_xfer(lo);
+               }
+       }
+       xfer_funcs[number] = NULL;
+       return 0;
+}
+
+EXPORT_SYMBOL(loop_register_transfer);
+EXPORT_SYMBOL(loop_unregister_transfer);
+
+int __init loop_init(void)
+{
+       int     i;
+
+       if ((max_loop < 1) || (max_loop > 256)) {
+               printk(KERN_WARNING "loop: invalid max_loop (must be between"
+                                   " 1 and 256), using default (8)\n");
+               max_loop = 8;
+       }
+
+       if (devfs_register_blkdev(MAJOR_NR, "loop", &lo_fops)) {
+               printk(KERN_WARNING "Unable to get major number %d for loop"
+                                   " device\n", MAJOR_NR);
+               return -EIO;
+       }
+
+       loop_dev = kmalloc(max_loop * sizeof(struct loop_device), GFP_KERNEL);
+       if (!loop_dev)
+               goto out_dev;
+
+       loop_sizes = kmalloc(max_loop * sizeof(int), GFP_KERNEL);
+       if (!loop_sizes)
+               goto out_sizes;
+
+       loop_blksizes = kmalloc(max_loop * sizeof(int), GFP_KERNEL);
+       if (!loop_blksizes)
+               goto out_blksizes;
+
+       loop_hardsizes = kmalloc(max_loop * sizeof(int), GFP_KERNEL);
+       if (!loop_hardsizes)
+               goto out_hardsizes;
+
+       blk_queue_make_request(BLK_DEFAULT_QUEUE(MAJOR_NR), loop_make_request);
+
+       for (i = 0; i < max_loop; i++) {
+               struct loop_device *lo = &loop_dev[i];
+               memset(lo, 0, sizeof(struct loop_device));
+               init_MUTEX(&lo->lo_ctl_mutex);
+               init_MUTEX_LOCKED(&lo->lo_sem);
+               lo->lo_number = i;
+               spin_lock_init(&lo->lo_lock);
+       }
+
+       memset(loop_sizes, 0, max_loop * sizeof(int));
+       memset(loop_blksizes, 0, max_loop * sizeof(int));
+       memset(loop_hardsizes, 0, max_loop * sizeof(int));
+       blk_size[MAJOR_NR] = loop_sizes;
+       blksize_size[MAJOR_NR] = loop_blksizes;
+       hardsect_size[MAJOR_NR] = loop_hardsizes;
+       for (i = 0; i < max_loop; i++)
+               register_disk(NULL, MKDEV(MAJOR_NR, i), 1, &lo_fops, 0);
+
+       for (i = 0; i < (sizeof(lo_prealloc) / sizeof(int)); i += 2) {
+               if (!lo_prealloc[i])
+                       continue;
+               if (lo_prealloc[i] < LO_PREALLOC_MIN)
+                       lo_prealloc[i] = LO_PREALLOC_MIN;
+               if (lo_prealloc[i] > LO_PREALLOC_MAX)
+                       lo_prealloc[i] = LO_PREALLOC_MAX;
+       }
+
+#if defined(IOCTL32_COMPATIBLE_PTR)
+       lock_kernel();
+       register_ioctl32_conversion(LOOP_SET_STATUS64, IOCTL32_COMPATIBLE_PTR);
+       register_ioctl32_conversion(LOOP_GET_STATUS64, IOCTL32_COMPATIBLE_PTR);
+       register_ioctl32_conversion(LOOP_MULTI_KEY_SETUP, IOCTL32_COMPATIBLE_PTR);
+       unlock_kernel();
+#endif
+
+       devfs_handle = devfs_mk_dir(NULL, "loop", NULL);
+       devfs_register_series(devfs_handle, "%u", max_loop, DEVFS_FL_DEFAULT,
+                             MAJOR_NR, 0,
+                             S_IFBLK | S_IRUSR | S_IWUSR | S_IRGRP,
+                             &lo_fops, NULL);
+
+#if CONFIG_BLK_DEV_LOOP_AES
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+       printk(KERN_INFO "loop: AES key scrubbing enabled\n");
+#endif
+#endif
+       printk(KERN_INFO "loop: loaded (max %d devices)\n", max_loop);
+       return 0;
+
+out_hardsizes:
+       kfree(loop_blksizes);
+out_blksizes:
+       kfree(loop_sizes);
+out_sizes:
+       kfree(loop_dev);
+out_dev:
+       if (devfs_unregister_blkdev(MAJOR_NR, "loop"))
+               printk(KERN_WARNING "loop: cannot unregister blkdev\n");
+       printk(KERN_ERR "loop: ran out of memory\n");
+       return -ENOMEM;
+}
+
+void loop_exit(void)
+{
+       devfs_unregister(devfs_handle);
+       if (devfs_unregister_blkdev(MAJOR_NR, "loop"))
+               printk(KERN_WARNING "loop: cannot unregister blkdev\n");
+
+       blk_size[MAJOR_NR] = 0;
+       blksize_size[MAJOR_NR] = 0;
+       hardsect_size[MAJOR_NR] = 0;
+       kfree(loop_dev);
+       kfree(loop_sizes);
+       kfree(loop_blksizes);
+       kfree(loop_hardsizes);
+
+#if defined(IOCTL32_COMPATIBLE_PTR)
+       lock_kernel();
+       unregister_ioctl32_conversion(LOOP_SET_STATUS64);
+       unregister_ioctl32_conversion(LOOP_GET_STATUS64);
+       unregister_ioctl32_conversion(LOOP_MULTI_KEY_SETUP);
+       unlock_kernel();
+#endif
+}
+
+module_init(loop_init);
+module_exit(loop_exit);
+
+#ifndef MODULE
+static int __init max_loop_setup(char *str)
+{
+       max_loop = simple_strtol(str, NULL, 0);
+       return 1;
+}
+
+__setup("max_loop=", max_loop_setup);
+#endif
+
+#if CONFIG_BLK_DEV_LOOP_KEYSCRUB
+void loop_add_keyscrub_fn(struct loop_device *lo, void (*fn)(void *), void *ptr)
+{
+    lo->lo_keyscrub_ptr = ptr;
+    wmb();
+    lo->lo_keyscrub_fn = fn;
+    wake_up_interruptible(&lo->lo_bh_wait);
+}
+EXPORT_SYMBOL(loop_add_keyscrub_fn);
+#endif
diff -urN linux-2.4.27-noloop/drivers/misc/Makefile linux-2.4.27-AES/drivers/misc/Makefile
--- linux-2.4.27-noloop/drivers/misc/Makefile   Sat Dec 30 00:07:22 2000
+++ linux-2.4.27-AES/drivers/misc/Makefile      Sun Aug 15 16:39:41 2004
@@ -9,7 +9,34 @@
 # parent makes..
 #
 
+.S.o:
+       $(CC) $(AFLAGS) $(AFLAGS_$@) -c $< -o $*.o
+
 O_TARGET := misc.o
+
+ifeq ($(CONFIG_BLK_DEV_LOOP_AES),y)
+AES_X86_ASM=n
+ifeq ($(CONFIG_X86),y)
+ifneq ($(CONFIG_X86_64),y)
+  AES_X86_ASM=y
+endif
+endif
+ifeq ($(AES_X86_ASM),y)
+  export-objs += crypto-ksym.o
+  obj-y       += aes-x86.o md5-x86.o crypto-ksym.o
+  AFLAGS_aes-x86.o := -DUSE_UNDERLINE=1
+else
+ifeq ($(CONFIG_X86_64),y)
+  export-objs += crypto-ksym.o
+  obj-y       += aes-amd64.o md5-amd64.o crypto-ksym.o
+  AFLAGS_aes-amd64.o := -DUSE_UNDERLINE=1
+else
+  export-objs += crypto-ksym.o
+  obj-y       += aes.o md5.o crypto-ksym.o
+  CFLAGS_aes.o := -DDATA_ALWAYS_ALIGNED=1
+endif
+endif
+endif
 
 include $(TOPDIR)/Rules.make
 
diff -urN linux-2.4.27-noloop/drivers/misc/aes-amd64.S linux-2.4.27-AES/drivers/misc/aes-amd64.S
--- linux-2.4.27-noloop/drivers/misc/aes-amd64.S        Thu Jan  1 02:00:00 1970
+++ linux-2.4.27-AES/drivers/misc/aes-amd64.S   Sun Aug 15 16:39:41 2004
@@ -0,0 +1,893 @@
+//
+// Copyright (c) 2001, Dr Brian Gladman <brg@gladman.uk.net>, Worcester, UK.
+// All rights reserved.
+//
+// TERMS
+//
+//  Redistribution and use in source and binary forms, with or without
+//  modification, are permitted subject to the following conditions:
+//
+//  1. Redistributions of source code must retain the above copyright
+//     notice, this list of conditions and the following disclaimer.
+//
+//  2. Redistributions in binary form must reproduce the above copyright
+//     notice, this list of conditions and the following disclaimer in the
+//     documentation and/or other materials provided with the distribution.
+//
+//  3. The copyright holder's name must not be used to endorse or promote
+//     any products derived from this software without his specific prior
+//     written permission.
+//
+//  This software is provided 'as is' with no express or implied warranties
+//  of correctness or fitness for purpose.
+
+// Modified by Jari Ruusu,  December 24 2001
+//  - Converted syntax to GNU CPP/assembler syntax
+//  - C programming interface converted back to "old" API
+//  - Minor portability cleanups and speed optimizations
+
+// Modified by Jari Ruusu,  April 11 2002
+//  - Added above copyright and terms to resulting object code so that
+//    binary distributions can avoid legal trouble
+
+// Modified by Jari Ruusu,  June 12 2004
+//  - Converted 32 bit x86 code to 64 bit AMD64 code
+//  - Re-wrote encrypt and decrypt code from scratch
+
+// An AES (Rijndael) implementation for the AMD64. This version only
+// implements the standard AES block length (128 bits, 16 bytes). This code
+// does not preserve the rax, rcx, rdx, rsi, rdi or r8-r11 registers or the
+// artihmetic status flags. However, the rbx, rbp and r12-r15 registers are
+// preserved across calls.
+
+// void aes_set_key(aes_context *cx, const unsigned char key[], const int key_len, const int f)
+// void aes_encrypt(const aes_context *cx, const unsigned char in_blk[], unsigned char out_blk[])
+// void aes_decrypt(const aes_context *cx, const unsigned char in_blk[], unsigned char out_blk[])
+
+#if defined(USE_UNDERLINE)
+# define aes_set_key _aes_set_key
+# define aes_encrypt _aes_encrypt
+# define aes_decrypt _aes_decrypt
+#endif
+#if !defined(ALIGN64BYTES)
+# define ALIGN64BYTES 64
+#endif
+
+       .file   "aes-amd64.S"
+       .globl  aes_set_key
+       .globl  aes_encrypt
+       .globl  aes_decrypt
+
+       .section .rodata
+copyright:
+       .ascii "    \000"
+       .ascii "Copyright (c) 2001, Dr Brian Gladman <brg@gladman.uk.net>, Worcester, UK.\000"
+       .ascii "All rights reserved.\000"
+       .ascii "    \000"
+       .ascii "TERMS\000"
+       .ascii "    \000"
+       .ascii " Redistribution and use in source and binary forms, with or without\000"
+       .ascii " modification, are permitted subject to the following conditions:\000"
+       .ascii "    \000"
+       .ascii " 1. Redistributions of source code must retain the above copyright\000"
+       .ascii "    notice, this list of conditions and the following disclaimer.\000"
+       .ascii "    \000"
+       .ascii " 2. Redistributions in binary form must reproduce the above copyright\000"
+       .ascii "    notice, this list of conditions and the following disclaimer in the\000"
+       .ascii "    documentation and/or other materials provided with the distribution.\000"
+       .ascii "    \000"
+       .ascii " 3. The copyright holder's name must not be used to endorse or promote\000"
+       .ascii "    any products derived from this software without his specific prior\000"
+       .ascii "    written permission.\000"
+       .ascii "    \000"
+       .ascii " This software is provided 'as is' with no express or implied warranties\000"
+       .ascii " of correctness or fitness for purpose.\000"
+       .ascii "    \000"
+
+#define tlen   1024    // length of each of 4 'xor' arrays (256 32-bit words)
+
+// offsets in context structure
+
+#define nkey   0       // key length, size 4
+#define nrnd   4       // number of rounds, size 4
+#define ekey   8       // encryption key schedule base address, size 256
+#define dkey   264     // decryption key schedule base address, size 256
+
+// This macro performs a forward encryption cycle. It is entered with
+// the first previous round column values in I1E, I2E, I3E and I4E and
+// exits with the final values OU1, OU2, OU3 and OU4 registers.
+
+#define fwd_rnd(p1,p2,I1E,I1B,I1H,I2E,I2B,I2H,I3E,I3B,I3R,I4E,I4B,I4R,OU1,OU2,OU3,OU4) \
+       movl    p2(%rbp),OU1            ;\
+       movl    p2+4(%rbp),OU2          ;\
+       movl    p2+8(%rbp),OU3          ;\
+       movl    p2+12(%rbp),OU4         ;\
+       movzbl  I1B,%edi                ;\
+       movzbl  I2B,%esi                ;\
+       movzbl  I3B,%r8d                ;\
+       movzbl  I4B,%r13d               ;\
+       shrl    $8,I3E                  ;\
+       shrl    $8,I4E                  ;\
+       xorl    p1(,%rdi,4),OU1         ;\
+       xorl    p1(,%rsi,4),OU2         ;\
+       xorl    p1(,%r8,4),OU3          ;\
+       xorl    p1(,%r13,4),OU4         ;\
+       movzbl  I2H,%esi                ;\
+       movzbl  I3B,%r8d                ;\
+       movzbl  I4B,%r13d               ;\
+       movzbl  I1H,%edi                ;\
+       shrl    $8,I3E                  ;\
+       shrl    $8,I4E                  ;\
+       xorl    p1+tlen(,%rsi,4),OU1    ;\
+       xorl    p1+tlen(,%r8,4),OU2     ;\
+       xorl    p1+tlen(,%r13,4),OU3    ;\
+       xorl    p1+tlen(,%rdi,4),OU4    ;\
+       shrl    $16,I1E                 ;\
+       shrl    $16,I2E                 ;\
+       movzbl  I3B,%r8d                ;\
+       movzbl  I4B,%r13d               ;\
+       movzbl  I1B,%edi                ;\
+       movzbl  I2B,%esi                ;\
+       xorl    p1+2*tlen(,%r8,4),OU1   ;\
+       xorl    p1+2*tlen(,%r13,4),OU2  ;\
+       xorl    p1+2*tlen(,%rdi,4),OU3  ;\
+       xorl    p1+2*tlen(,%rsi,4),OU4  ;\
+       shrl    $8,I4E                  ;\
+       movzbl  I1H,%edi                ;\
+       movzbl  I2H,%esi                ;\
+       shrl    $8,I3E                  ;\
+       xorl    p1+3*tlen(,I4R,4),OU1   ;\
+       xorl    p1+3*tlen(,%rdi,4),OU2  ;\
+       xorl    p1+3*tlen(,%rsi,4),OU3  ;\
+       xorl    p1+3*tlen(,I3R,4),OU4
+
+// This macro performs an inverse encryption cycle. It is entered with
+// the first previous round column values in I1E, I2E, I3E and I4E and
+// exits with the final values OU1, OU2, OU3 and OU4 registers.
+
+#define inv_rnd(p1,p2,I1E,I1B,I1R,I2E,I2B,I2R,I3E,I3B,I3H,I4E,I4B,I4H,OU1,OU2,OU3,OU4) \
+       movl    p2+12(%rbp),OU4         ;\
+       movl    p2+8(%rbp),OU3          ;\
+       movl    p2+4(%rbp),OU2          ;\
+       movl    p2(%rbp),OU1            ;\
+       movzbl  I4B,%edi                ;\
+       movzbl  I3B,%esi                ;\
+       movzbl  I2B,%r8d                ;\
+       movzbl  I1B,%r13d               ;\
+       shrl    $8,I2E                  ;\
+       shrl    $8,I1E                  ;\
+       xorl    p1(,%rdi,4),OU4         ;\
+       xorl    p1(,%rsi,4),OU3         ;\
+       xorl    p1(,%r8,4),OU2          ;\
+       xorl    p1(,%r13,4),OU1         ;\
+       movzbl  I3H,%esi                ;\
+       movzbl  I2B,%r8d                ;\
+       movzbl  I1B,%r13d               ;\
+       movzbl  I4H,%edi                ;\
+       shrl    $8,I2E                  ;\
+       shrl    $8,I1E                  ;\
+       xorl    p1+tlen(,%rsi,4),OU4    ;\
+       xorl    p1+tlen(,%r8,4),OU3     ;\
+       xorl    p1+tlen(,%r13,4),OU2    ;\
+       xorl    p1+tlen(,%rdi,4),OU1    ;\
+       shrl    $16,I4E                 ;\
+       shrl    $16,I3E                 ;\
+       movzbl  I2B,%r8d                ;\
+       movzbl  I1B,%r13d               ;\
+       movzbl  I4B,%edi                ;\
+       movzbl  I3B,%esi                ;\
+       xorl    p1+2*tlen(,%r8,4),OU4   ;\
+       xorl    p1+2*tlen(,%r13,4),OU3  ;\
+       xorl    p1+2*tlen(,%rdi,4),OU2  ;\
+       xorl    p1+2*tlen(,%rsi,4),OU1  ;\
+       shrl    $8,I1E                  ;\
+       movzbl  I4H,%edi                ;\
+       movzbl  I3H,%esi                ;\
+       shrl    $8,I2E                  ;\
+       xorl    p1+3*tlen(,I1R,4),OU4   ;\
+       xorl    p1+3*tlen(,%rdi,4),OU3  ;\
+       xorl    p1+3*tlen(,%rsi,4),OU2  ;\
+       xorl    p1+3*tlen(,I2R,4),OU1
+
+// AES (Rijndael) Encryption Subroutine
+
+// rdi = pointer to AES context
+// rsi = pointer to input plaintext bytes
+// rdx = pointer to output ciphertext bytes
+
+       .text
+       .align  ALIGN64BYTES
+aes_encrypt:
+       movl    (%rsi),%eax             // read in plaintext
+       movl    4(%rsi),%ecx
+       movl    8(%rsi),%r10d
+       movl    12(%rsi),%r11d
+
+       pushq   %rbp
+       leaq    ekey+16(%rdi),%rbp      // encryption key pointer
+       movq    %rdx,%r9                // pointer to out block
+       movl    nrnd(%rdi),%edx         // number of rounds
+       pushq   %rbx
+       pushq   %r13
+       pushq   %r14
+       pushq   %r15
+
+       xorl    -16(%rbp),%eax          // xor in first round key
+       xorl    -12(%rbp),%ecx
+       xorl    -8(%rbp),%r10d
+       xorl    -4(%rbp),%r11d
+
+       subl    $10,%edx
+       je      aes_15
+       addq    $32,%rbp
+       subl    $2,%edx
+       je      aes_13
+       addq    $32,%rbp
+
+       fwd_rnd(aes_ft_tab,-64,%eax,%al,%ah,%ecx,%cl,%ch,%r10d,%r10b,%r10,%r11d,%r11b,%r11,%ebx,%edx,%r14d,%r15d)
+       fwd_rnd(aes_ft_tab,-48,%ebx,%bl,%bh,%edx,%dl,%dh,%r14d,%r14b,%r14,%r15d,%r15b,%r15,%eax,%ecx,%r10d,%r11d)
+       jmp     aes_13
+       .align  ALIGN64BYTES
+aes_13:        fwd_rnd(aes_ft_tab,-32,%eax,%al,%ah,%ecx,%cl,%ch,%r10d,%r10b,%r10,%r11d,%r11b,%r11,%ebx,%edx,%r14d,%r15d)
+       fwd_rnd(aes_ft_tab,-16,%ebx,%bl,%bh,%edx,%dl,%dh,%r14d,%r14b,%r14,%r15d,%r15b,%r15,%eax,%ecx,%r10d,%r11d)
+       jmp     aes_15
+       .align  ALIGN64BYTES
+aes_15:        fwd_rnd(aes_ft_tab,0,  %eax,%al,%ah,%ecx,%cl,%ch,%r10d,%r10b,%r10,%r11d,%r11b,%r11,%ebx,%edx,%r14d,%r15d)
+       fwd_rnd(aes_ft_tab,16, %ebx,%bl,%bh,%edx,%dl,%dh,%r14d,%r14b,%r14,%r15d,%r15b,%r15,%eax,%ecx,%r10d,%r11d)
+       fwd_rnd(aes_ft_tab,32, %eax,%al,%ah,%ecx,%cl,%ch,%r10d,%r10b,%r10,%r11d,%r11b,%r11,%ebx,%edx,%r14d,%r15d)
+       fwd_rnd(aes_ft_tab,48, %ebx,%bl,%bh,%edx,%dl,%dh,%r14d,%r14b,%r14,%r15d,%r15b,%r15,%eax,%ecx,%r10d,%r11d)
+       fwd_rnd(aes_ft_tab,64, %eax,%al,%ah,%ecx,%cl,%ch,%r10d,%r10b,%r10,%r11d,%r11b,%r11,%ebx,%edx,%r14d,%r15d)
+       fwd_rnd(aes_ft_tab,80, %ebx,%bl,%bh,%edx,%dl,%dh,%r14d,%r14b,%r14,%r15d,%r15b,%r15,%eax,%ecx,%r10d,%r11d)
+       fwd_rnd(aes_ft_tab,96, %eax,%al,%ah,%ecx,%cl,%ch,%r10d,%r10b,%r10,%r11d,%r11b,%r11,%ebx,%edx,%r14d,%r15d)
+       fwd_rnd(aes_ft_tab,112,%ebx,%bl,%bh,%edx,%dl,%dh,%r14d,%r14b,%r14,%r15d,%r15b,%r15,%eax,%ecx,%r10d,%r11d)
+       fwd_rnd(aes_ft_tab,128,%eax,%al,%ah,%ecx,%cl,%ch,%r10d,%r10b,%r10,%r11d,%r11b,%r11,%ebx,%edx,%r14d,%r15d)
+       fwd_rnd(aes_fl_tab,144,%ebx,%bl,%bh,%edx,%dl,%dh,%r14d,%r14b,%r14,%r15d,%r15b,%r15,%eax,%ecx,%r10d,%r11d)
+
+       popq    %r15
+       popq    %r14
+       popq    %r13
+       popq    %rbx
+       popq    %rbp
+
+       movl    %eax,(%r9)              // move final values to the output array.
+       movl    %ecx,4(%r9)
+       movl    %r10d,8(%r9)
+       movl    %r11d,12(%r9)
+       ret
+
+// AES (Rijndael) Decryption Subroutine
+
+// rdi = pointer to AES context
+// rsi = pointer to input ciphertext bytes
+// rdx = pointer to output plaintext bytes
+
+       .align  ALIGN64BYTES
+aes_decrypt:
+       movl    12(%rsi),%eax           // read in ciphertext
+       movl    8(%rsi),%ecx
+       movl    4(%rsi),%r10d
+       movl    (%rsi),%r11d
+
+       pushq   %rbp
+       leaq    dkey+16(%rdi),%rbp      // decryption key pointer
+       movq    %rdx,%r9                // pointer to out block
+       movl    nrnd(%rdi),%edx         // number of rounds
+       pushq   %rbx
+       pushq   %r13
+       pushq   %r14
+       pushq   %r15
+
+       xorl    -4(%rbp),%eax           // xor in first round key
+       xorl    -8(%rbp),%ecx
+       xorl    -12(%rbp),%r10d
+       xorl    -16(%rbp),%r11d
+
+       subl    $10,%edx
+       je      aes_25
+       addq    $32,%rbp
+       subl    $2,%edx
+       je      aes_23
+       addq    $32,%rbp
+
+       inv_rnd(aes_it_tab,-64,%r11d,%r11b,%r11,%r10d,%r10b,%r10,%ecx,%cl,%ch,%eax,%al,%ah,%r15d,%r14d,%edx,%ebx)
+       inv_rnd(aes_it_tab,-48,%r15d,%r15b,%r15,%r14d,%r14b,%r14,%edx,%dl,%dh,%ebx,%bl,%bh,%r11d,%r10d,%ecx,%eax)
+       jmp     aes_23
+       .align  ALIGN64BYTES
+aes_23:        inv_rnd(aes_it_tab,-32,%r11d,%r11b,%r11,%r10d,%r10b,%r10,%ecx,%cl,%ch,%eax,%al,%ah,%r15d,%r14d,%edx,%ebx)
+       inv_rnd(aes_it_tab,-16,%r15d,%r15b,%r15,%r14d,%r14b,%r14,%edx,%dl,%dh,%ebx,%bl,%bh,%r11d,%r10d,%ecx,%eax)
+       jmp     aes_25
+       .align  ALIGN64BYTES
+aes_25:        inv_rnd(aes_it_tab,0,  %r11d,%r11b,%r11,%r10d,%r10b,%r10,%ecx,%cl,%ch,%eax,%al,%ah,%r15d,%r14d,%edx,%ebx)
+       inv_rnd(aes_it_tab,16, %r15d,%r15b,%r15,%r14d,%r14b,%r14,%edx,%dl,%dh,%ebx,%bl,%bh,%r11d,%r10d,%ecx,%eax)
+       inv_rnd(aes_it_tab,32, %r11d,%r11b,%r11,%r10d,%r10b,%r10,%ecx,%cl,%ch,%eax,%al,%ah,%r15d,%r14d,%edx,%ebx)
+       inv_rnd(aes_it_tab,48, %r15d,%r15b,%r15,%r14d,%r14b,%r14,%edx,%dl,%dh,%ebx,%bl,%bh,%r11d,%r10d,%ecx,%eax)
+       inv_rnd(aes_it_tab,64, %r11d,%r11b,%r11,%r10d,%r10b,%r10,%ecx,%cl,%ch,%eax,%al,%ah,%r15d,%r14d,%edx,%ebx)
+       inv_rnd(aes_it_tab,80, %r15d,%r15b,%r15,%r14d,%r14b,%r14,%edx,%dl,%dh,%ebx,%bl,%bh,%r11d,%r10d,%ecx,%eax)
+       inv_rnd(aes_it_tab,96, %r11d,%r11b,%r11,%r10d,%r10b,%r10,%ecx,%cl,%ch,%eax,%al,%ah,%r15d,%r14d,%edx,%ebx)
+       inv_rnd(aes_it_tab,112,%r15d,%r15b,%r15,%r14d,%r14b,%r14,%edx,%dl,%dh,%ebx,%bl,%bh,%r11d,%r10d,%ecx,%eax)
+       inv_rnd(aes_it_tab,128,%r11d,%r11b,%r11,%r10d,%r10b,%r10,%ecx,%cl,%ch,%eax,%al,%ah,%r15d,%r14d,%edx,%ebx)
+       inv_rnd(aes_il_tab,144,%r15d,%r15b,%r15,%r14d,%r14b,%r14,%edx,%dl,%dh,%ebx,%bl,%bh,%r11d,%r10d,%ecx,%eax)
+
+       popq    %r15
+       popq    %r14
+       popq    %r13
+       popq    %rbx
+       popq    %rbp
+
+       movl    %eax,12(%r9)            // move final values to the output array.
+       movl    %ecx,8(%r9)
+       movl    %r10d,4(%r9)
+       movl    %r11d,(%r9)
+       ret
+
+// AES (Rijndael) Key Schedule Subroutine
+
+// This macro performs a column mixing operation on an input 32-bit
+// word to give a 32-bit result. It uses each of the 4 bytes in the
+// the input column to index 4 different tables of 256 32-bit words
+// that are xored together to form the output value.
+
+#define mix_col(p1)                     \
+       movzbl  %bl,%ecx                ;\
+       movl    p1(,%rcx,4),%eax        ;\
+       movzbl  %bh,%ecx                ;\
+       ror     $16,%ebx                ;\
+       xorl    p1+tlen(,%rcx,4),%eax   ;\
+       movzbl  %bl,%ecx                ;\
+       xorl    p1+2*tlen(,%rcx,4),%eax ;\
+       movzbl  %bh,%ecx                ;\
+       xorl    p1+3*tlen(,%rcx,4),%eax
+
+// Key Schedule Macros
+
+#define ksc4(p1)                        \
+       rol     $24,%ebx                ;\
+       mix_col(aes_fl_tab)             ;\
+       ror     $8,%ebx                 ;\
+       xorl    4*p1+aes_rcon_tab,%eax  ;\
+       xorl    %eax,%esi               ;\
+       xorl    %esi,%ebp               ;\
+       movl    %esi,16*p1(%rdi)        ;\
+       movl    %ebp,16*p1+4(%rdi)      ;\
+       xorl    %ebp,%edx               ;\
+       xorl    %edx,%ebx               ;\
+       movl    %edx,16*p1+8(%rdi)      ;\
+       movl    %ebx,16*p1+12(%rdi)
+
+#define ksc6(p1)                        \
+       rol     $24,%ebx                ;\
+       mix_col(aes_fl_tab)             ;\
+       ror     $8,%ebx                 ;\
+       xorl    4*p1+aes_rcon_tab,%eax  ;\
+       xorl    24*p1-24(%rdi),%eax     ;\
+       movl    %eax,24*p1(%rdi)        ;\
+       xorl    24*p1-20(%rdi),%eax     ;\
+       movl    %eax,24*p1+4(%rdi)      ;\
+       xorl    %eax,%esi               ;\
+       xorl    %esi,%ebp               ;\
+       movl    %esi,24*p1+8(%rdi)      ;\
+       movl    %ebp,24*p1+12(%rdi)     ;\
+       xorl    %ebp,%edx               ;\
+       xorl    %edx,%ebx               ;\
+       movl    %edx,24*p1+16(%rdi)     ;\
+       movl    %ebx,24*p1+20(%rdi)
+
+#define ksc8(p1)                        \
+       rol     $24,%ebx                ;\
+       mix_col(aes_fl_tab)             ;\
+       ror     $8,%ebx                 ;\
+       xorl    4*p1+aes_rcon_tab,%eax  ;\
+       xorl    32*p1-32(%rdi),%eax     ;\
+       movl    %eax,32*p1(%rdi)        ;\
+       xorl    32*p1-28(%rdi),%eax     ;\
+       movl    %eax,32*p1+4(%rdi)      ;\
+       xorl    32*p1-24(%rdi),%eax     ;\
+       movl    %eax,32*p1+8(%rdi)      ;\
+       xorl    32*p1-20(%rdi),%eax     ;\
+       movl    %eax,32*p1+12(%rdi)     ;\
+       pushq   %rbx                    ;\
+       movl    %eax,%ebx               ;\
+       mix_col(aes_fl_tab)             ;\
+       popq    %rbx                    ;\
+       xorl    %eax,%esi               ;\
+       xorl    %esi,%ebp               ;\
+       movl    %esi,32*p1+16(%rdi)     ;\
+       movl    %ebp,32*p1+20(%rdi)     ;\
+       xorl    %ebp,%edx               ;\
+       xorl    %edx,%ebx               ;\
+       movl    %edx,32*p1+24(%rdi)     ;\
+       movl    %ebx,32*p1+28(%rdi)
+
+// rdi = pointer to AES context
+// rsi = pointer to key bytes
+// rdx = key length, bytes or bits
+// rcx = ed_flag, 1=encrypt only, 0=both encrypt and decrypt
+
+       .align  ALIGN64BYTES
+aes_set_key:
+       pushfq
+       pushq   %rbp
+       pushq   %rbx
+
+       movq    %rcx,%r11               // ed_flg
+       movq    %rdx,%rcx               // key length
+       movq    %rdi,%r10               // AES context
+
+       cmpl    $128,%ecx
+       jb      aes_30
+       shrl    $3,%ecx
+aes_30:        cmpl    $32,%ecx
+       je      aes_32
+       cmpl    $24,%ecx
+       je      aes_32
+       movl    $16,%ecx
+aes_32:        shrl    $2,%ecx
+       movl    %ecx,nkey(%r10)
+       leaq    6(%rcx),%rax            // 10/12/14 for 4/6/8 32-bit key length
+       movl    %eax,nrnd(%r10)
+       leaq    ekey(%r10),%rdi         // key position in AES context
+       cld
+       movl    %ecx,%eax               // save key length in eax
+       rep ;   movsl                   // words in the key schedule
+       movl    -4(%rsi),%ebx           // put some values in registers
+       movl    -8(%rsi),%edx           // to allow faster code
+       movl    -12(%rsi),%ebp
+       movl    -16(%rsi),%esi
+
+       cmpl    $4,%eax                 // jump on key size
+       je      aes_36
+       cmpl    $6,%eax
+       je      aes_35
+
+       ksc8(0)
+       ksc8(1)
+       ksc8(2)
+       ksc8(3)
+       ksc8(4)
+       ksc8(5)
+       ksc8(6)
+       jmp     aes_37
+aes_35:        ksc6(0)
+       ksc6(1)
+       ksc6(2)
+       ksc6(3)
+       ksc6(4)
+       ksc6(5)
+       ksc6(6)
+       ksc6(7)
+       jmp     aes_37
+aes_36:        ksc4(0)
+       ksc4(1)
+       ksc4(2)
+       ksc4(3)
+       ksc4(4)
+       ksc4(5)
+       ksc4(6)
+       ksc4(7)
+       ksc4(8)
+       ksc4(9)
+aes_37:        cmpl    $0,%r11d                // ed_flg
+       jne     aes_39
+
+// compile decryption key schedule from encryption schedule - reverse
+// order and do mix_column operation on round keys except first and last
+
+       movl    nrnd(%r10),%eax         // kt = cx->d_key + nc * cx->Nrnd
+       shl     $2,%rax
+       leaq    dkey(%r10,%rax,4),%rdi
+       leaq    ekey(%r10),%rsi         // kf = cx->e_key
+
+       movsq                           // copy first round key (unmodified)
+       movsq
+       subq    $32,%rdi
+       movl    $1,%r9d
+aes_38:                                        // do mix column on each column of
+       lodsl                           // each round key
+       movl    %eax,%ebx
+       mix_col(aes_im_tab)
+       stosl
+       lodsl
+       movl    %eax,%ebx
+       mix_col(aes_im_tab)
+       stosl
+       lodsl
+       movl    %eax,%ebx
+       mix_col(aes_im_tab)
+       stosl
+       lodsl
+       movl    %eax,%ebx
+       mix_col(aes_im_tab)
+       stosl
+       subq    $32,%rdi
+
+       incl    %r9d
+       cmpl    nrnd(%r10),%r9d
+       jb      aes_38
+
+       movsq                           // copy last round key (unmodified)
+       movsq
+aes_39:        popq    %rbx
+       popq    %rbp
+       popfq
+       ret
+
+
+// finite field multiplies by {02}, {04} and {08}
+
+#define f2(x)  ((x<<1)^(((x>>7)&1)*0x11b))
+#define f4(x)  ((x<<2)^(((x>>6)&1)*0x11b)^(((x>>6)&2)*0x11b))
+#define f8(x)  ((x<<3)^(((x>>5)&1)*0x11b)^(((x>>5)&2)*0x11b)^(((x>>5)&4)*0x11b))
+
+// finite field multiplies required in table generation
+
+#define f3(x)  (f2(x) ^ x)
+#define f9(x)  (f8(x) ^ x)
+#define fb(x)  (f8(x) ^ f2(x) ^ x)
+#define fd(x)  (f8(x) ^ f4(x) ^ x)
+#define fe(x)  (f8(x) ^ f4(x) ^ f2(x))
+
+// These defines generate the forward table entries
+
+#define u0(x)  ((f3(x) << 24) | (x << 16) | (x << 8) | f2(x))
+#define u1(x)  ((x << 24) | (x << 16) | (f2(x) << 8) | f3(x))
+#define u2(x)  ((x << 24) | (f2(x) << 16) | (f3(x) << 8) | x)
+#define u3(x)  ((f2(x) << 24) | (f3(x) << 16) | (x << 8) | x)
+
+// These defines generate the inverse table entries
+
+#define v0(x)  ((fb(x) << 24) | (fd(x) << 16) | (f9(x) << 8) | fe(x))
+#define v1(x)  ((fd(x) << 24) | (f9(x) << 16) | (fe(x) << 8) | fb(x))
+#define v2(x)  ((f9(x) << 24) | (fe(x) << 16) | (fb(x) << 8) | fd(x))
+#define v3(x)  ((fe(x) << 24) | (fb(x) << 16) | (fd(x) << 8) | f9(x))
+
+// These defines generate entries for the last round tables
+
+#define w0(x)  (x)
+#define w1(x)  (x <<  8)
+#define w2(x)  (x << 16)
+#define w3(x)  (x << 24)
+
+// macro to generate inverse mix column tables (needed for the key schedule)
+
+#define im_data0(p1) \
+       .long   p1(0x00),p1(0x01),p1(0x02),p1(0x03),p1(0x04),p1(0x05),p1(0x06),p1(0x07) ;\
+       .long   p1(0x08),p1(0x09),p1(0x0a),p1(0x0b),p1(0x0c),p1(0x0d),p1(0x0e),p1(0x0f) ;\
+       .long   p1(0x10),p1(0x11),p1(0x12),p1(0x13),p1(0x14),p1(0x15),p1(0x16),p1(0x17) ;\
+       .long   p1(0x18),p1(0x19),p1(0x1a),p1(0x1b),p1(0x1c),p1(0x1d),p1(0x1e),p1(0x1f)
+#define im_data1(p1) \
+       .long   p1(0x20),p1(0x21),p1(0x22),p1(0x23),p1(0x24),p1(0x25),p1(0x26),p1(0x27) ;\
+       .long   p1(0x28),p1(0x29),p1(0x2a),p1(0x2b),p1(0x2c),p1(0x2d),p1(0x2e),p1(0x2f) ;\
+       .long   p1(0x30),p1(0x31),p1(0x32),p1(0x33),p1(0x34),p1(0x35),p1(0x36),p1(0x37) ;\
+       .long   p1(0x38),p1(0x39),p1(0x3a),p1(0x3b),p1(0x3c),p1(0x3d),p1(0x3e),p1(0x3f)
+#define im_data2(p1) \
+       .long   p1(0x40),p1(0x41),p1(0x42),p1(0x43),p1(0x44),p1(0x45),p1(0x46),p1(0x47) ;\
+       .long   p1(0x48),p1(0x49),p1(0x4a),p1(0x4b),p1(0x4c),p1(0x4d),p1(0x4e),p1(0x4f) ;\
+       .long   p1(0x50),p1(0x51),p1(0x52),p1(0x53),p1(0x54),p1(0x55),p1(0x56),p1(0x57) ;\
+       .long   p1(0x58),p1(0x59),p1(0x5a),p1(0x5b),p1(0x5c),p1(0x5d),p1(0x5e),p1(0x5f)
+#define im_data3(p1) \
+       .long   p1(0x60),p1(0x61),p1(0x62),p1(0x63),p1(0x64),p1(0x65),p1(0x66),p1(0x67) ;\
+       .long   p1(0x68),p1(0x69),p1(0x6a),p1(0x6b),p1(0x6c),p1(0x6d),p1(0x6e),p1(0x6f) ;\
+       .long   p1(0x70),p1(0x71),p1(0x72),p1(0x73),p1(0x74),p1(0x75),p1(0x76),p1(0x77) ;\
+       .long   p1(0x78),p1(0x79),p1(0x7a),p1(0x7b),p1(0x7c),p1(0x7d),p1(0x7e),p1(0x7f)
+#define im_data4(p1) \
+       .long   p1(0x80),p1(0x81),p1(0x82),p1(0x83),p1(0x84),p1(0x85),p1(0x86),p1(0x87) ;\
+       .long   p1(0x88),p1(0x89),p1(0x8a),p1(0x8b),p1(0x8c),p1(0x8d),p1(0x8e),p1(0x8f) ;\
+       .long   p1(0x90),p1(0x91),p1(0x92),p1(0x93),p1(0x94),p1(0x95),p1(0x96),p1(0x97) ;\
+       .long   p1(0x98),p1(0x99),p1(0x9a),p1(0x9b),p1(0x9c),p1(0x9d),p1(0x9e),p1(0x9f)
+#define im_data5(p1) \
+       .long   p1(0xa0),p1(0xa1),p1(0xa2),p1(0xa3),p1(0xa4),p1(0xa5),p1(0xa6),p1(0xa7) ;\
+       .long   p1(0xa8),p1(0xa9),p1(0xaa),p1(0xab),p1(0xac),p1(0xad),p1(0xae),p1(0xaf) ;\
+       .long   p1(0xb0),p1(0xb1),p1(0xb2),p1(0xb3),p1(0xb4),p1(0xb5),p1(0xb6),p1(0xb7) ;\
+       .long   p1(0xb8),p1(0xb9),p1(0xba),p1(0xbb),p1(0xbc),p1(0xbd),p1(0xbe),p1(0xbf)
+#define im_data6(p1) \
+       .long   p1(0xc0),p1(0xc1),p1(0xc2),p1(0xc3),p1(0xc4),p1(0xc5),p1(0xc6),p1(0xc7) ;\
+       .long   p1(0xc8),p1(0xc9),p1(0xca),p1(0xcb),p1(0xcc),p1(0xcd),p1(0xce),p1(0xcf) ;\
+       .long   p1(0xd0),p1(0xd1),p1(0xd2),p1(0xd3),p1(0xd4),p1(0xd5),p1(0xd6),p1(0xd7) ;\
+       .long   p1(0xd8),p1(0xd9),p1(0xda),p1(0xdb),p1(0xdc),p1(0xdd),p1(0xde),p1(0xdf)
+#define im_data7(p1) \
+       .long   p1(0xe0),p1(0xe1),p1(0xe2),p1(0xe3),p1(0xe4),p1(0xe5),p1(0xe6),p1(0xe7) ;\
+       .long   p1(0xe8),p1(0xe9),p1(0xea),p1(0xeb),p1(0xec),p1(0xed),p1(0xee),p1(0xef) ;\
+       .long   p1(0xf0),p1(0xf1),p1(0xf2),p1(0xf3),p1(0xf4),p1(0xf5),p1(0xf6),p1(0xf7) ;\
+       .long   p1(0xf8),p1(0xf9),p1(0xfa),p1(0xfb),p1(0xfc),p1(0xfd),p1(0xfe),p1(0xff)
+
+// S-box data - 256 entries
+
+#define sb_data0(p1) \
+       .long   p1(0x63),p1(0x7c),p1(0x77),p1(0x7b),p1(0xf2),p1(0x6b),p1(0x6f),p1(0xc5) ;\
+       .long   p1(0x30),p1(0x01),p1(0x67),p1(0x2b),p1(0xfe),p1(0xd7),p1(0xab),p1(0x76) ;\
+       .long   p1(0xca),p1(0x82),p1(0xc9),p1(0x7d),p1(0xfa),p1(0x59),p1(0x47),p1(0xf0) ;\
+       .long   p1(0xad),p1(0xd4),p1(0xa2),p1(0xaf),p1(0x9c),p1(0xa4),p1(0x72),p1(0xc0)
+#define sb_data1(p1) \
+       .long   p1(0xb7),p1(0xfd),p1(0x93),p1(0x26),p1(0x36),p1(0x3f),p1(0xf7),p1(0xcc) ;\
+       .long   p1(0x34),p1(0xa5),p1(0xe5),p1(0xf1),p1(0x71),p1(0xd8),p1(0x31),p1(0x15) ;\
+       .long   p1(0x04),p1(0xc7),p1(0x23),p1(0xc3),p1(0x18),p1(0x96),p1(0x05),p1(0x9a) ;\
+       .long   p1(0x07),p1(0x12),p1(0x80),p1(0xe2),p1(0xeb),p1(0x27),p1(0xb2),p1(0x75)
+#define sb_data2(p1) \
+       .long   p1(0x09),p1(0x83),p1(0x2c),p1(0x1a),p1(0x1b),p1(0x6e),p1(0x5a),p1(0xa0) ;\
+       .long   p1(0x52),p1(0x3b),p1(0xd6),p1(0xb3),p1(0x29),p1(0xe3),p1(0x2f),p1(0x84) ;\
+       .long   p1(0x53),p1(0xd1),p1(0x00),p1(0xed),p1(0x20),p1(0xfc),p1(0xb1),p1(0x5b) ;\
+       .long   p1(0x6a),p1(0xcb),p1(0xbe),p1(0x39),p1(0x4a),p1(0x4c),p1(0x58),p1(0xcf)
+#define sb_data3(p1) \
+       .long   p1(0xd0),p1(0xef),p1(0xaa),p1(0xfb),p1(0x43),p1(0x4d),p1(0x33),p1(0x85) ;\
+       .long   p1(0x45),p1(0xf9),p1(0x02),p1(0x7f),p1(0x50),p1(0x3c),p1(0x9f),p1(0xa8) ;\
+       .long   p1(0x51),p1(0xa3),p1(0x40),p1(0x8f),p1(0x92),p1(0x9d),p1(0x38),p1(0xf5) ;\
+       .long   p1(0xbc),p1(0xb6),p1(0xda),p1(0x21),p1(0x10),p1(0xff),p1(0xf3),p1(0xd2)
+#define sb_data4(p1) \
+       .long   p1(0xcd),p1(0x0c),p1(0x13),p1(0xec),p1(0x5f),p1(0x97),p1(0x44),p1(0x17) ;\
+       .long   p1(0xc4),p1(0xa7),p1(0x7e),p1(0x3d),p1(0x64),p1(0x5d),p1(0x19),p1(0x73) ;\
+       .long   p1(0x60),p1(0x81),p1(0x4f),p1(0xdc),p1(0x22),p1(0x2a),p1(0x90),p1(0x88) ;\
+       .long   p1(0x46),p1(0xee),p1(0xb8),p1(0x14),p1(0xde),p1(0x5e),p1(0x0b),p1(0xdb)
+#define sb_data5(p1) \
+       .long   p1(0xe0),p1(0x32),p1(0x3a),p1(0x0a),p1(0x49),p1(0x06),p1(0x24),p1(0x5c) ;\
+       .long   p1(0xc2),p1(0xd3),p1(0xac),p1(0x62),p1(0x91),p1(0x95),p1(0xe4),p1(0x79) ;\
+       .long   p1(0xe7),p1(0xc8),p1(0x37),p1(0x6d),p1(0x8d),p1(0xd5),p1(0x4e),p1(0xa9) ;\
+       .long   p1(0x6c),p1(0x56),p1(0xf4),p1(0xea),p1(0x65),p1(0x7a),p1(0xae),p1(0x08)
+#define sb_data6(p1) \
+       .long   p1(0xba),p1(0x78),p1(0x25),p1(0x2e),p1(0x1c),p1(0xa6),p1(0xb4),p1(0xc6) ;\
+       .long   p1(0xe8),p1(0xdd),p1(0x74),p1(0x1f),p1(0x4b),p1(0xbd),p1(0x8b),p1(0x8a) ;\
+       .long   p1(0x70),p1(0x3e),p1(0xb5),p1(0x66),p1(0x48),p1(0x03),p1(0xf6),p1(0x0e) ;\
+       .long   p1(0x61),p1(0x35),p1(0x57),p1(0xb9),p1(0x86),p1(0xc1),p1(0x1d),p1(0x9e)
+#define sb_data7(p1) \
+       .long   p1(0xe1),p1(0xf8),p1(0x98),p1(0x11),p1(0x69),p1(0xd9),p1(0x8e),p1(0x94) ;\
+       .long   p1(0x9b),p1(0x1e),p1(0x87),p1(0xe9),p1(0xce),p1(0x55),p1(0x28),p1(0xdf) ;\
+       .long   p1(0x8c),p1(0xa1),p1(0x89),p1(0x0d),p1(0xbf),p1(0xe6),p1(0x42),p1(0x68) ;\
+       .long   p1(0x41),p1(0x99),p1(0x2d),p1(0x0f),p1(0xb0),p1(0x54),p1(0xbb),p1(0x16)
+
+// Inverse S-box data - 256 entries
+
+#define ib_data0(p1) \
+       .long   p1(0x52),p1(0x09),p1(0x6a),p1(0xd5),p1(0x30),p1(0x36),p1(0xa5),p1(0x38) ;\
+       .long   p1(0xbf),p1(0x40),p1(0xa3),p1(0x9e),p1(0x81),p1(0xf3),p1(0xd7),p1(0xfb) ;\
+       .long   p1(0x7c),p1(0xe3),p1(0x39),p1(0x82),p1(0x9b),p1(0x2f),p1(0xff),p1(0x87) ;\
+       .long   p1(0x34),p1(0x8e),p1(0x43),p1(0x44),p1(0xc4),p1(0xde),p1(0xe9),p1(0xcb)
+#define ib_data1(p1) \
+       .long   p1(0x54),p1(0x7b),p1(0x94),p1(0x32),p1(0xa6),p1(0xc2),p1(0x23),p1(0x3d) ;\
+       .long   p1(0xee),p1(0x4c),p1(0x95),p1(0x0b),p1(0x42),p1(0xfa),p1(0xc3),p1(0x4e) ;\
+       .long   p1(0x08),p1(0x2e),p1(0xa1),p1(0x66),p1(0x28),p1(0xd9),p1(0x24),p1(0xb2) ;\
+       .long   p1(0x76),p1(0x5b),p1(0xa2),p1(0x49),p1(0x6d),p1(0x8b),p1(0xd1),p1(0x25)
+#define ib_data2(p1) \
+       .long   p1(0x72),p1(0xf8),p1(0xf6),p1(0x64),p1(0x86),p1(0x68),p1(0x98),p1(0x16) ;\
+       .long   p1(0xd4),p1(0xa4),p1(0x5c),p1(0xcc),p1(0x5d),p1(0x65),p1(0xb6),p1(0x92) ;\
+       .long   p1(0x6c),p1(0x70),p1(0x48),p1(0x50),p1(0xfd),p1(0xed),p1(0xb9),p1(0xda) ;\
+       .long   p1(0x5e),p1(0x15),p1(0x46),p1(0x57),p1(0xa7),p1(0x8d),p1(0x9d),p1(0x84)
+#define ib_data3(p1) \
+       .long   p1(0x90),p1(0xd8),p1(0xab),p1(0x00),p1(0x8c),p1(0xbc),p1(0xd3),p1(0x0a) ;\
+       .long   p1(0xf7),p1(0xe4),p1(0x58),p1(0x05),p1(0xb8),p1(0xb3),p1(0x45),p1(0x06) ;\
+       .long   p1(0xd0),p1(0x2c),p1(0x1e),p1(0x8f),p1(0xca),p1(0x3f),p1(0x0f),p1(0x02) ;\
+       .long   p1(0xc1),p1(0xaf),p1(0xbd),p1(0x03),p1(0x01),p1(0x13),p1(0x8a),p1(0x6b)
+#define ib_data4(p1) \
+       .long   p1(0x3a),p1(0x91),p1(0x11),p1(0x41),p1(0x4f),p1(0x67),p1(0xdc),p1(0xea) ;\
+       .long   p1(0x97),p1(0xf2),p1(0xcf),p1(0xce),p1(0xf0),p1(0xb4),p1(0xe6),p1(0x73) ;\
+       .long   p1(0x96),p1(0xac),p1(0x74),p1(0x22),p1(0xe7),p1(0xad),p1(0x35),p1(0x85) ;\
+       .long   p1(0xe2),p1(0xf9),p1(0x37),p1(0xe8),p1(0x1c),p1(0x75),p1(0xdf),p1(0x6e)
+#define ib_data5(p1) \
+       .long   p1(0x47),p1(0xf1),p1(0x1a),p1(0x71),p1(0x1d),p1(0x29),p1(0xc5),p1(0x89) ;\
+       .long   p1(0x6f),p1(0xb7),p1(0x62),p1(0x0e),p1(0xaa),p1(0x18),p1(0xbe),p1(0x1b) ;\
+       .long   p1(0xfc),p1(0x56),p1(0x3e),p1(0x4b),p1(0xc6),p1(0xd2),p1(0x79),p1(0x20) ;\
+       .long   p1(0x9a),p1(0xdb),p1(0xc0),p1(0xfe),p1(0x78),p1(0xcd),p1(0x5a),p1(0xf4)
+#define ib_data6(p1) \
+       .long   p1(0x1f),p1(0xdd),p1(0xa8),p1(0x33),p1(0x88),p1(0x07),p1(0xc7),p1(0x31) ;\
+       .long   p1(0xb1),p1(0x12),p1(0x10),p1(0x59),p1(0x27),p1(0x80),p1(0xec),p1(0x5f) ;\
+       .long   p1(0x60),p1(0x51),p1(0x7f),p1(0xa9),p1(0x19),p1(0xb5),p1(0x4a),p1(0x0d) ;\
+       .long   p1(0x2d),p1(0xe5),p1(0x7a),p1(0x9f),p1(0x93),p1(0xc9),p1(0x9c),p1(0xef)
+#define ib_data7(p1) \
+       .long   p1(0xa0),p1(0xe0),p1(0x3b),p1(0x4d),p1(0xae),p1(0x2a),p1(0xf5),p1(0xb0) ;\
+       .long   p1(0xc8),p1(0xeb),p1(0xbb),p1(0x3c),p1(0x83),p1(0x53),p1(0x99),p1(0x61) ;\
+       .long   p1(0x17),p1(0x2b),p1(0x04),p1(0x7e),p1(0xba),p1(0x77),p1(0xd6),p1(0x26) ;\
+       .long   p1(0xe1),p1(0x69),p1(0x14),p1(0x63),p1(0x55),p1(0x21),p1(0x0c),p1(0x7d)
+
+// The rcon_table (needed for the key schedule)
+//
+// Here is original Dr Brian Gladman's source code:
+//     _rcon_tab:
+//     %assign x   1
+//     %rep 29
+//         dd  x
+//     %assign x f2(x)
+//     %endrep
+//
+// Here is precomputed output (it's more portable this way):
+
+       .section .rodata
+       .align  ALIGN64BYTES
+aes_rcon_tab:
+       .long   0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80
+       .long   0x1b,0x36,0x6c,0xd8,0xab,0x4d,0x9a,0x2f
+       .long   0x5e,0xbc,0x63,0xc6,0x97,0x35,0x6a,0xd4
+       .long   0xb3,0x7d,0xfa,0xef,0xc5
+
+// The forward xor tables
+
+       .align  ALIGN64BYTES
+aes_ft_tab:
+       sb_data0(u0)
+       sb_data1(u0)
+       sb_data2(u0)
+       sb_data3(u0)
+       sb_data4(u0)
+       sb_data5(u0)
+       sb_data6(u0)
+       sb_data7(u0)
+
+       sb_data0(u1)
+       sb_data1(u1)
+       sb_data2(u1)
+       sb_data3(u1)
+       sb_data4(u1)
+       sb_data5(u1)
+       sb_data6(u1)
+       sb_data7(u1)
+
+       sb_data0(u2)
+       sb_data1(u2)
+       sb_data2(u2)
+       sb_data3(u2)
+       sb_data4(u2)
+       sb_data5(u2)
+       sb_data6(u2)
+       sb_data7(u2)
+
+       sb_data0(u3)
+       sb_data1(u3)
+       sb_data2(u3)
+       sb_data3(u3)
+       sb_data4(u3)
+       sb_data5(u3)
+       sb_data6(u3)
+       sb_data7(u3)
+
+       .align  ALIGN64BYTES
+aes_fl_tab:
+       sb_data0(w0)
+       sb_data1(w0)
+       sb_data2(w0)
+       sb_data3(w0)
+       sb_data4(w0)
+       sb_data5(w0)
+       sb_data6(w0)
+       sb_data7(w0)
+
+       sb_data0(w1)
+       sb_data1(w1)
+       sb_data2(w1)
+       sb_data3(w1)
+       sb_data4(w1)
+       sb_data5(w1)
+       sb_data6(w1)
+       sb_data7(w1)
+
+       sb_data0(w2)
+       sb_data1(w2)
+       sb_data2(w2)
+       sb_data3(w2)
+       sb_data4(w2)
+       sb_data5(w2)
+       sb_data6(w2)
+       sb_data7(w2)
+
+       sb_data0(w3)
+       sb_data1(w3)
+       sb_data2(w3)
+       sb_data3(w3)
+       sb_data4(w3)
+       sb_data5(w3)
+       sb_data6(w3)
+       sb_data7(w3)
+
+// The inverse xor tables
+
+       .align  ALIGN64BYTES
+aes_it_tab:
+       ib_data0(v0)
+       ib_data1(v0)
+       ib_data2(v0)
+       ib_data3(v0)
+       ib_data4(v0)
+       ib_data5(v0)
+       ib_data6(v0)
+       ib_data7(v0)
+
+       ib_data0(v1)
+       ib_data1(v1)
+       ib_data2(v1)
+       ib_data3(v1)
+       ib_data4(v1)
+       ib_data5(v1)
+       ib_data6(v1)
+       ib_data7(v1)
+
+       ib_data0(v2)
+       ib_data1(v2)
+       ib_data2(v2)
+       ib_data3(v2)
+       ib_data4(v2)
+       ib_data5(v2)
+       ib_data6(v2)
+       ib_data7(v2)
+
+       ib_data0(v3)
+       ib_data1(v3)
+       ib_data2(v3)
+       ib_data3(v3)
+       ib_data4(v3)
+       ib_data5(v3)
+       ib_data6(v3)
+       ib_data7(v3)
+
+       .align  ALIGN64BYTES
+aes_il_tab:
+       ib_data0(w0)
+       ib_data1(w0)
+       ib_data2(w0)
+       ib_data3(w0)
+       ib_data4(w0)
+       ib_data5(w0)
+       ib_data6(w0)
+       ib_data7(w0)
+
+       ib_data0(w1)
+       ib_data1(w1)
+       ib_data2(w1)
+       ib_data3(w1)
+       ib_data4(w1)
+       ib_data5(w1)
+       ib_data6(w1)
+       ib_data7(w1)
+
+       ib_data0(w2)
+       ib_data1(w2)
+       ib_data2(w2)
+       ib_data3(w2)
+       ib_data4(w2)
+       ib_data5(w2)
+       ib_data6(w2)
+       ib_data7(w2)
+
+       ib_data0(w3)
+       ib_data1(w3)
+       ib_data2(w3)
+       ib_data3(w3)
+       ib_data4(w3)
+       ib_data5(w3)
+       ib_data6(w3)
+       ib_data7(w3)
+
+// The inverse mix column tables
+
+       .align  ALIGN64BYTES
+aes_im_tab:
+       im_data0(v0)
+       im_data1(v0)
+       im_data2(v0)
+       im_data3(v0)
+       im_data4(v0)
+       im_data5(v0)
+       im_data6(v0)
+       im_data7(v0)
+
+       im_data0(v1)
+       im_data1(v1)
+       im_data2(v1)
+       im_data3(v1)
+       im_data4(v1)
+       im_data5(v1)
+       im_data6(v1)
+       im_data7(v1)
+
+       im_data0(v2)
+       im_data1(v2)
+       im_data2(v2)
+       im_data3(v2)
+       im_data4(v2)
+       im_data5(v2)
+       im_data6(v2)
+       im_data7(v2)
+
+       im_data0(v3)
+       im_data1(v3)
+       im_data2(v3)
+       im_data3(v3)
+       im_data4(v3)
+       im_data5(v3)
+       im_data6(v3)
+       im_data7(v3)
diff -urN linux-2.4.27-noloop/drivers/misc/aes-x86.S linux-2.4.27-AES/drivers/misc/aes-x86.S
--- linux-2.4.27-noloop/drivers/misc/aes-x86.S  Thu Jan  1 02:00:00 1970
+++ linux-2.4.27-AES/drivers/misc/aes-x86.S     Sun Aug 15 16:39:41 2004
@@ -0,0 +1,922 @@
+//
+// Copyright (c) 2001, Dr Brian Gladman <brg@gladman.uk.net>, Worcester, UK.
+// All rights reserved.
+//
+// TERMS
+//
+//  Redistribution and use in source and binary forms, with or without
+//  modification, are permitted subject to the following conditions:
+//
+//  1. Redistributions of source code must retain the above copyright
+//     notice, this list of conditions and the following disclaimer.
+//
+//  2. Redistributions in binary form must reproduce the above copyright
+//     notice, this list of conditions and the following disclaimer in the
+//     documentation and/or other materials provided with the distribution.
+//
+//  3. The copyright holder's name must not be used to endorse or promote
+//     any products derived from this software without his specific prior
+//     written permission.
+//
+//  This software is provided 'as is' with no express or implied warranties
+//  of correctness or fitness for purpose.
+
+// Modified by Jari Ruusu,  December 24 2001
+//  - Converted syntax to GNU CPP/assembler syntax
+//  - C programming interface converted back to "old" API
+//  - Minor portability cleanups and speed optimizations
+
+// Modified by Jari Ruusu,  April 11 2002
+//  - Added above copyright and terms to resulting object code so that
+//    binary distributions can avoid legal trouble
+
+// An AES (Rijndael) implementation for x86 compatible processors. This
+// version uses i386 instruction set but instruction scheduling is optimized
+// for Pentium-2. This version only implements the standard AES block length
+// (128 bits, 16 bytes). This code does not preserve the eax, ecx or edx
+// registers or the artihmetic status flags. However, the ebx, esi, edi, and
+// ebp registers are preserved across calls.
+
+// void aes_set_key(aes_context *cx, const unsigned char key[], const int key_len, const int f)
+// void aes_encrypt(const aes_context *cx, const unsigned char in_blk[], unsigned char out_blk[])
+// void aes_decrypt(const aes_context *cx, const unsigned char in_blk[], unsigned char out_blk[])
+
+#if defined(USE_UNDERLINE)
+# define aes_set_key _aes_set_key
+# define aes_encrypt _aes_encrypt
+# define aes_decrypt _aes_decrypt
+#endif
+#if !defined(ALIGN32BYTES)
+# define ALIGN32BYTES 32
+#endif
+
+       .file   "aes-x86.S"
+       .globl  aes_set_key
+       .globl  aes_encrypt
+       .globl  aes_decrypt
+
+       .text
+copyright:
+       .ascii "    \000"
+       .ascii "Copyright (c) 2001, Dr Brian Gladman <brg@gladman.uk.net>, Worcester, UK.\000"
+       .ascii "All rights reserved.\000"
+       .ascii "    \000"
+       .ascii "TERMS\000"
+       .ascii "    \000"
+       .ascii " Redistribution and use in source and binary forms, with or without\000"
+       .ascii " modification, are permitted subject to the following conditions:\000"
+       .ascii "    \000"
+       .ascii " 1. Redistributions of source code must retain the above copyright\000"
+       .ascii "    notice, this list of conditions and the following disclaimer.\000"
+       .ascii "    \000"
+       .ascii " 2. Redistributions in binary form must reproduce the above copyright\000"
+       .ascii "    notice, this list of conditions and the following disclaimer in the\000"
+       .ascii "    documentation and/or other materials provided with the distribution.\000"
+       .ascii "    \000"
+       .ascii " 3. The copyright holder's name must not be used to endorse or promote\000"
+       .ascii "    any products derived from this software without his specific prior\000"
+       .ascii "    written permission.\000"
+       .ascii "    \000"
+       .ascii " This software is provided 'as is' with no express or implied warranties\000"
+       .ascii " of correctness or fitness for purpose.\000"
+       .ascii "    \000"
+
+#define tlen   1024    // length of each of 4 'xor' arrays (256 32-bit words)
+
+// offsets to parameters with one register pushed onto stack
+
+#define ctx    8       // AES context structure
+#define in_blk 12      // input byte array address parameter
+#define out_blk        16      // output byte array address parameter
+
+// offsets in context structure
+
+#define nkey   0       // key length, size 4
+#define nrnd   4       // number of rounds, size 4
+#define ekey   8       // encryption key schedule base address, size 256
+#define dkey   264     // decryption key schedule base address, size 256
+
+// This macro performs a forward encryption cycle. It is entered with
+// the first previous round column values in %eax, %ebx, %esi and %edi and
+// exits with the final values in the same registers.
+
+#define fwd_rnd(p1,p2)                  \
+       mov     %ebx,(%esp)             ;\
+       movzbl  %al,%edx                ;\
+       mov     %eax,%ecx               ;\
+       mov     p2(%ebp),%eax           ;\
+       mov     %edi,4(%esp)            ;\
+       mov     p2+12(%ebp),%edi        ;\
+       xor     p1(,%edx,4),%eax        ;\
+       movzbl  %ch,%edx                ;\
+       shr     $16,%ecx                ;\
+       mov     p2+4(%ebp),%ebx         ;\
+       xor     p1+tlen(,%edx,4),%edi   ;\
+       movzbl  %cl,%edx                ;\
+       movzbl  %ch,%ecx                ;\
+       xor     p1+3*tlen(,%ecx,4),%ebx ;\
+       mov     %esi,%ecx               ;\
+       mov     p1+2*tlen(,%edx,4),%esi ;\
+       movzbl  %cl,%edx                ;\
+       xor     p1(,%edx,4),%esi        ;\
+       movzbl  %ch,%edx                ;\
+       shr     $16,%ecx                ;\
+       xor     p1+tlen(,%edx,4),%ebx   ;\
+       movzbl  %cl,%edx                ;\
+       movzbl  %ch,%ecx                ;\
+       xor     p1+2*tlen(,%edx,4),%eax ;\
+       mov     (%esp),%edx             ;\
+       xor     p1+3*tlen(,%ecx,4),%edi ;\
+       movzbl  %dl,%ecx                ;\
+       xor     p2+8(%ebp),%esi         ;\
+       xor     p1(,%ecx,4),%ebx        ;\
+       movzbl  %dh,%ecx                ;\
+       shr     $16,%edx                ;\
+       xor     p1+tlen(,%ecx,4),%eax   ;\
+       movzbl  %dl,%ecx                ;\
+       movzbl  %dh,%edx                ;\
+       xor     p1+2*tlen(,%ecx,4),%edi ;\
+       mov     4(%esp),%ecx            ;\
+       xor     p1+3*tlen(,%edx,4),%esi ;\
+       movzbl  %cl,%edx                ;\
+       xor     p1(,%edx,4),%edi        ;\
+       movzbl  %ch,%edx                ;\
+       shr     $16,%ecx                ;\
+       xor     p1+tlen(,%edx,4),%esi   ;\
+       movzbl  %cl,%edx                ;\
+       movzbl  %ch,%ecx                ;\
+       xor     p1+2*tlen(,%edx,4),%ebx ;\
+       xor     p1+3*tlen(,%ecx,4),%eax
+
+// This macro performs an inverse encryption cycle. It is entered with
+// the first previous round column values in %eax, %ebx, %esi and %edi and
+// exits with the final values in the same registers.
+
+#define inv_rnd(p1,p2)                  \
+       movzbl  %al,%edx                ;\
+       mov     %ebx,(%esp)             ;\
+       mov     %eax,%ecx               ;\
+       mov     p2(%ebp),%eax           ;\
+       mov     %edi,4(%esp)            ;\
+       mov     p2+4(%ebp),%ebx         ;\
+       xor     p1(,%edx,4),%eax        ;\
+       movzbl  %ch,%edx                ;\
+       shr     $16,%ecx                ;\
+       mov     p2+12(%ebp),%edi        ;\
+       xor     p1+tlen(,%edx,4),%ebx   ;\
+       movzbl  %cl,%edx                ;\
+       movzbl  %ch,%ecx                ;\
+       xor     p1+3*tlen(,%ecx,4),%edi ;\
+       mov     %esi,%ecx               ;\
+       mov     p1+2*tlen(,%edx,4),%esi ;\
+       movzbl  %cl,%edx                ;\
+       xor     p1(,%edx,4),%esi        ;\
+       movzbl  %ch,%edx                ;\
+       shr     $16,%ecx                ;\
+       xor     p1+tlen(,%edx,4),%edi   ;\
+       movzbl  %cl,%edx                ;\
+       movzbl  %ch,%ecx                ;\
+       xor     p1+2*tlen(,%edx,4),%eax ;\
+       mov     (%esp),%edx             ;\
+       xor     p1+3*tlen(,%ecx,4),%ebx ;\
+       movzbl  %dl,%ecx                ;\
+       xor     p2+8(%ebp),%esi         ;\
+       xor     p1(,%ecx,4),%ebx        ;\
+       movzbl  %dh,%ecx                ;\
+       shr     $16,%edx                ;\
+       xor     p1+tlen(,%ecx,4),%esi   ;\
+       movzbl  %dl,%ecx                ;\
+       movzbl  %dh,%edx                ;\
+       xor     p1+2*tlen(,%ecx,4),%edi ;\
+       mov     4(%esp),%ecx            ;\
+       xor     p1+3*tlen(,%edx,4),%eax ;\
+       movzbl  %cl,%edx                ;\
+       xor     p1(,%edx,4),%edi        ;\
+       movzbl  %ch,%edx                ;\
+       shr     $16,%ecx                ;\
+       xor     p1+tlen(,%edx,4),%eax   ;\
+       movzbl  %cl,%edx                ;\
+       movzbl  %ch,%ecx                ;\
+       xor     p1+2*tlen(,%edx,4),%ebx ;\
+       xor     p1+3*tlen(,%ecx,4),%esi
+
+// AES (Rijndael) Encryption Subroutine
+
+       .text
+       .align  ALIGN32BYTES
+aes_encrypt:
+       push    %ebp
+       mov     ctx(%esp),%ebp          // pointer to context
+       mov     in_blk(%esp),%ecx
+       push    %ebx
+       push    %esi
+       push    %edi
+       mov     nrnd(%ebp),%edx         // number of rounds
+       lea     ekey+16(%ebp),%ebp      // key pointer
+
+// input four columns and xor in first round key
+
+       mov     (%ecx),%eax
+       mov     4(%ecx),%ebx
+       mov     8(%ecx),%esi
+       mov     12(%ecx),%edi
+       xor     -16(%ebp),%eax
+       xor     -12(%ebp),%ebx
+       xor     -8(%ebp),%esi
+       xor     -4(%ebp),%edi
+
+       sub     $8,%esp                 // space for register saves on stack
+
+       sub     $10,%edx
+       je      aes_15
+       add     $32,%ebp
+       sub     $2,%edx
+       je      aes_13
+       add     $32,%ebp
+
+       fwd_rnd(aes_ft_tab,-64)         // 14 rounds for 256-bit key
+       fwd_rnd(aes_ft_tab,-48)
+aes_13:        fwd_rnd(aes_ft_tab,-32)         // 12 rounds for 192-bit key
+       fwd_rnd(aes_ft_tab,-16)
+aes_15:        fwd_rnd(aes_ft_tab,0)           // 10 rounds for 128-bit key
+       fwd_rnd(aes_ft_tab,16)
+       fwd_rnd(aes_ft_tab,32)
+       fwd_rnd(aes_ft_tab,48)
+       fwd_rnd(aes_ft_tab,64)
+       fwd_rnd(aes_ft_tab,80)
+       fwd_rnd(aes_ft_tab,96)
+       fwd_rnd(aes_ft_tab,112)
+       fwd_rnd(aes_ft_tab,128)
+       fwd_rnd(aes_fl_tab,144)         // last round uses a different table
+
+// move final values to the output array.
+
+       mov     out_blk+20(%esp),%ebp
+       add     $8,%esp
+       mov     %eax,(%ebp)
+       mov     %ebx,4(%ebp)
+       mov     %esi,8(%ebp)
+       mov     %edi,12(%ebp)
+       pop     %edi
+       pop     %esi
+       pop     %ebx
+       pop     %ebp
+       ret
+
+
+// AES (Rijndael) Decryption Subroutine
+
+       .align  ALIGN32BYTES
+aes_decrypt:
+       push    %ebp
+       mov     ctx(%esp),%ebp          // pointer to context
+       mov     in_blk(%esp),%ecx
+       push    %ebx
+       push    %esi
+       push    %edi
+       mov     nrnd(%ebp),%edx         // number of rounds
+       lea     dkey+16(%ebp),%ebp      // key pointer
+
+// input four columns and xor in first round key
+
+       mov     (%ecx),%eax
+       mov     4(%ecx),%ebx
+       mov     8(%ecx),%esi
+       mov     12(%ecx),%edi
+       xor     -16(%ebp),%eax
+       xor     -12(%ebp),%ebx
+       xor     -8(%ebp),%esi
+       xor     -4(%ebp),%edi
+
+       sub     $8,%esp                 // space for register saves on stack
+
+       sub     $10,%edx
+       je      aes_25
+       add     $32,%ebp
+       sub     $2,%edx
+       je      aes_23
+       add     $32,%ebp
+
+       inv_rnd(aes_it_tab,-64)         // 14 rounds for 256-bit key
+       inv_rnd(aes_it_tab,-48)
+aes_23:        inv_rnd(aes_it_tab,-32)         // 12 rounds for 192-bit key
+       inv_rnd(aes_it_tab,-16)
+aes_25:        inv_rnd(aes_it_tab,0)           // 10 rounds for 128-bit key
+       inv_rnd(aes_it_tab,16)
+       inv_rnd(aes_it_tab,32)
+       inv_rnd(aes_it_tab,48)
+       inv_rnd(aes_it_tab,64)
+       inv_rnd(aes_it_tab,80)
+       inv_rnd(aes_it_tab,96)
+       inv_rnd(aes_it_tab,112)
+       inv_rnd(aes_it_tab,128)
+       inv_rnd(aes_il_tab,144)         // last round uses a different table
+
+// move final values to the output array.
+
+       mov     out_blk+20(%esp),%ebp
+       add     $8,%esp
+       mov     %eax,(%ebp)
+       mov     %ebx,4(%ebp)
+       mov     %esi,8(%ebp)
+       mov     %edi,12(%ebp)
+       pop     %edi
+       pop     %esi
+       pop     %ebx
+       pop     %ebp
+       ret
+
+// AES (Rijndael) Key Schedule Subroutine
+
+// input/output parameters
+
+#define aes_cx 12      // AES context
+#define in_key 16      // key input array address
+#define key_ln 20      // key length, bytes (16,24,32) or bits (128,192,256)
+#define ed_flg 24      // 0=create both encr/decr keys, 1=create encr key only
+
+// offsets for locals
+
+#define cnt    -4
+#define slen   8
+
+// This macro performs a column mixing operation on an input 32-bit
+// word to give a 32-bit result. It uses each of the 4 bytes in the
+// the input column to index 4 different tables of 256 32-bit words
+// that are xored together to form the output value.
+
+#define mix_col(p1)                     \
+       movzbl  %bl,%ecx                ;\
+       mov     p1(,%ecx,4),%eax        ;\
+       movzbl  %bh,%ecx                ;\
+       ror     $16,%ebx                ;\
+       xor     p1+tlen(,%ecx,4),%eax   ;\
+       movzbl  %bl,%ecx                ;\
+       xor     p1+2*tlen(,%ecx,4),%eax ;\
+       movzbl  %bh,%ecx                ;\
+       xor     p1+3*tlen(,%ecx,4),%eax
+
+// Key Schedule Macros
+
+#define ksc4(p1)                        \
+       rol     $24,%ebx                ;\
+       mix_col(aes_fl_tab)             ;\
+       ror     $8,%ebx                 ;\
+       xor     4*p1+aes_rcon_tab,%eax  ;\
+       xor     %eax,%esi               ;\
+       xor     %esi,%ebp               ;\
+       mov     %esi,16*p1(%edi)        ;\
+       mov     %ebp,16*p1+4(%edi)      ;\
+       xor     %ebp,%edx               ;\
+       xor     %edx,%ebx               ;\
+       mov     %edx,16*p1+8(%edi)      ;\
+       mov     %ebx,16*p1+12(%edi)
+
+#define ksc6(p1)                        \
+       rol     $24,%ebx                ;\
+       mix_col(aes_fl_tab)             ;\
+       ror     $8,%ebx                 ;\
+       xor     4*p1+aes_rcon_tab,%eax  ;\
+       xor     24*p1-24(%edi),%eax     ;\
+       mov     %eax,24*p1(%edi)        ;\
+       xor     24*p1-20(%edi),%eax     ;\
+       mov     %eax,24*p1+4(%edi)      ;\
+       xor     %eax,%esi               ;\
+       xor     %esi,%ebp               ;\
+       mov     %esi,24*p1+8(%edi)      ;\
+       mov     %ebp,24*p1+12(%edi)     ;\
+       xor     %ebp,%edx               ;\
+       xor     %edx,%ebx               ;\
+       mov     %edx,24*p1+16(%edi)     ;\
+       mov     %ebx,24*p1+20(%edi)
+
+#define ksc8(p1)                        \
+       rol     $24,%ebx                ;\
+       mix_col(aes_fl_tab)             ;\
+       ror     $8,%ebx                 ;\
+       xor     4*p1+aes_rcon_tab,%eax  ;\
+       xor     32*p1-32(%edi),%eax     ;\
+       mov     %eax,32*p1(%edi)        ;\
+       xor     32*p1-28(%edi),%eax     ;\
+       mov     %eax,32*p1+4(%edi)      ;\
+       xor     32*p1-24(%edi),%eax     ;\
+       mov     %eax,32*p1+8(%edi)      ;\
+       xor     32*p1-20(%edi),%eax     ;\
+       mov     %eax,32*p1+12(%edi)     ;\
+       push    %ebx                    ;\
+       mov     %eax,%ebx               ;\
+       mix_col(aes_fl_tab)             ;\
+       pop     %ebx                    ;\
+       xor     %eax,%esi               ;\
+       xor     %esi,%ebp               ;\
+       mov     %esi,32*p1+16(%edi)     ;\
+       mov     %ebp,32*p1+20(%edi)     ;\
+       xor     %ebp,%edx               ;\
+       xor     %edx,%ebx               ;\
+       mov     %edx,32*p1+24(%edi)     ;\
+       mov     %ebx,32*p1+28(%edi)
+
+       .align  ALIGN32BYTES
+aes_set_key:
+       pushfl
+       push    %ebp
+       mov     %esp,%ebp
+       sub     $slen,%esp
+       push    %ebx
+       push    %esi
+       push    %edi
+
+       mov     aes_cx(%ebp),%edx       // edx -> AES context
+
+       mov     key_ln(%ebp),%ecx       // key length
+       cmpl    $128,%ecx
+       jb      aes_30
+       shr     $3,%ecx
+aes_30:        cmpl    $32,%ecx
+       je      aes_32
+       cmpl    $24,%ecx
+       je      aes_32
+       mov     $16,%ecx
+aes_32:        shr     $2,%ecx
+       mov     %ecx,nkey(%edx)
+
+       lea     6(%ecx),%eax            // 10/12/14 for 4/6/8 32-bit key length
+       mov     %eax,nrnd(%edx)
+
+       mov     in_key(%ebp),%esi       // key input array
+       lea     ekey(%edx),%edi         // key position in AES context
+       cld
+       push    %ebp
+       mov     %ecx,%eax               // save key length in eax
+       rep ;   movsl                   // words in the key schedule
+       mov     -4(%esi),%ebx           // put some values in registers
+       mov     -8(%esi),%edx           // to allow faster code
+       mov     -12(%esi),%ebp
+       mov     -16(%esi),%esi
+
+       cmpl    $4,%eax                 // jump on key size
+       je      aes_36
+       cmpl    $6,%eax
+       je      aes_35
+
+       ksc8(0)
+       ksc8(1)
+       ksc8(2)
+       ksc8(3)
+       ksc8(4)
+       ksc8(5)
+       ksc8(6)
+       jmp     aes_37
+aes_35:        ksc6(0)
+       ksc6(1)
+       ksc6(2)
+       ksc6(3)
+       ksc6(4)
+       ksc6(5)
+       ksc6(6)
+       ksc6(7)
+       jmp     aes_37
+aes_36:        ksc4(0)
+       ksc4(1)
+       ksc4(2)
+       ksc4(3)
+       ksc4(4)
+       ksc4(5)
+       ksc4(6)
+       ksc4(7)
+       ksc4(8)
+       ksc4(9)
+aes_37:        pop     %ebp
+       mov     aes_cx(%ebp),%edx       // edx -> AES context
+       cmpl    $0,ed_flg(%ebp)
+       jne     aes_39
+
+// compile decryption key schedule from encryption schedule - reverse
+// order and do mix_column operation on round keys except first and last
+
+       mov     nrnd(%edx),%eax         // kt = cx->d_key + nc * cx->Nrnd
+       shl     $2,%eax
+       lea     dkey(%edx,%eax,4),%edi
+       lea     ekey(%edx),%esi         // kf = cx->e_key
+
+       movsl                           // copy first round key (unmodified)
+       movsl
+       movsl
+       movsl
+       sub     $32,%edi
+       movl    $1,cnt(%ebp)
+aes_38:                                        // do mix column on each column of
+       lodsl                           // each round key
+       mov     %eax,%ebx
+       mix_col(aes_im_tab)
+       stosl
+       lodsl
+       mov     %eax,%ebx
+       mix_col(aes_im_tab)
+       stosl
+       lodsl
+       mov     %eax,%ebx
+       mix_col(aes_im_tab)
+       stosl
+       lodsl
+       mov     %eax,%ebx
+       mix_col(aes_im_tab)
+       stosl
+       sub     $32,%edi
+
+       incl    cnt(%ebp)
+       mov     cnt(%ebp),%eax
+       cmp     nrnd(%edx),%eax
+       jb      aes_38
+
+       movsl                           // copy last round key (unmodified)
+       movsl
+       movsl
+       movsl
+aes_39:        pop     %edi
+       pop     %esi
+       pop     %ebx
+       mov     %ebp,%esp
+       pop     %ebp
+       popfl
+       ret
+
+
+// finite field multiplies by {02}, {04} and {08}
+
+#define f2(x)  ((x<<1)^(((x>>7)&1)*0x11b))
+#define f4(x)  ((x<<2)^(((x>>6)&1)*0x11b)^(((x>>6)&2)*0x11b))
+#define f8(x)  ((x<<3)^(((x>>5)&1)*0x11b)^(((x>>5)&2)*0x11b)^(((x>>5)&4)*0x11b))
+
+// finite field multiplies required in table generation
+
+#define f3(x)  (f2(x) ^ x)
+#define f9(x)  (f8(x) ^ x)
+#define fb(x)  (f8(x) ^ f2(x) ^ x)
+#define fd(x)  (f8(x) ^ f4(x) ^ x)
+#define fe(x)  (f8(x) ^ f4(x) ^ f2(x))
+
+// These defines generate the forward table entries
+
+#define u0(x)  ((f3(x) << 24) | (x << 16) | (x << 8) | f2(x))
+#define u1(x)  ((x << 24) | (x << 16) | (f2(x) << 8) | f3(x))
+#define u2(x)  ((x << 24) | (f2(x) << 16) | (f3(x) << 8) | x)
+#define u3(x)  ((f2(x) << 24) | (f3(x) << 16) | (x << 8) | x)
+
+// These defines generate the inverse table entries
+
+#define v0(x)  ((fb(x) << 24) | (fd(x) << 16) | (f9(x) << 8) | fe(x))
+#define v1(x)  ((fd(x) << 24) | (f9(x) << 16) | (fe(x) << 8) | fb(x))
+#define v2(x)  ((f9(x) << 24) | (fe(x) << 16) | (fb(x) << 8) | fd(x))
+#define v3(x)  ((fe(x) << 24) | (fb(x) << 16) | (fd(x) << 8) | f9(x))
+
+// These defines generate entries for the last round tables
+
+#define w0(x)  (x)
+#define w1(x)  (x <<  8)
+#define w2(x)  (x << 16)
+#define w3(x)  (x << 24)
+
+// macro to generate inverse mix column tables (needed for the key schedule)
+
+#define im_data0(p1) \
+       .long   p1(0x00),p1(0x01),p1(0x02),p1(0x03),p1(0x04),p1(0x05),p1(0x06),p1(0x07) ;\
+       .long   p1(0x08),p1(0x09),p1(0x0a),p1(0x0b),p1(0x0c),p1(0x0d),p1(0x0e),p1(0x0f) ;\
+       .long   p1(0x10),p1(0x11),p1(0x12),p1(0x13),p1(0x14),p1(0x15),p1(0x16),p1(0x17) ;\
+       .long   p1(0x18),p1(0x19),p1(0x1a),p1(0x1b),p1(0x1c),p1(0x1d),p1(0x1e),p1(0x1f)
+#define im_data1(p1) \
+       .long   p1(0x20),p1(0x21),p1(0x22),p1(0x23),p1(0x24),p1(0x25),p1(0x26),p1(0x27) ;\
+       .long   p1(0x28),p1(0x29),p1(0x2a),p1(0x2b),p1(0x2c),p1(0x2d),p1(0x2e),p1(0x2f) ;\
+       .long   p1(0x30),p1(0x31),p1(0x32),p1(0x33),p1(0x34),p1(0x35),p1(0x36),p1(0x37) ;\
+       .long   p1(0x38),p1(0x39),p1(0x3a),p1(0x3b),p1(0x3c),p1(0x3d),p1(0x3e),p1(0x3f)
+#define im_data2(p1) \
+       .long   p1(0x40),p1(0x41),p1(0x42),p1(0x43),p1(0x44),p1(0x45),p1(0x46),p1(0x47) ;\
+       .long   p1(0x48),p1(0x49),p1(0x4a),p1(0x4b),p1(0x4c),p1(0x4d),p1(0x4e),p1(0x4f) ;\
+       .long   p1(0x50),p1(0x51),p1(0x52),p1(0x53),p1(0x54),p1(0x55),p1(0x56),p1(0x57) ;\
+       .long   p1(0x58),p1(0x59),p1(0x5a),p1(0x5b),p1(0x5c),p1(0x5d),p1(0x5e),p1(0x5f)
+#define im_data3(p1) \
+       .long   p1(0x60),p1(0x61),p1(0x62),p1(0x63),p1(0x64),p1(0x65),p1(0x66),p1(0x67) ;\
+       .long   p1(0x68),p1(0x69),p1(0x6a),p1(0x6b),p1(0x6c),p1(0x6d),p1(0x6e),p1(0x6f) ;\
+       .long   p1(0x70),p1(0x71),p1(0x72),p1(0x73),p1(0x74),p1(0x75),p1(0x76),p1(0x77) ;\
+       .long   p1(0x78),p1(0x79),p1(0x7a),p1(0x7b),p1(0x7c),p1(0x7d),p1(0x7e),p1(0x7f)
+#define im_data4(p1) \
+       .long   p1(0x80),p1(0x81),p1(0x82),p1(0x83),p1(0x84),p1(0x85),p1(0x86),p1(0x87) ;\
+       .long   p1(0x88),p1(0x89),p1(0x8a),p1(0x8b),p1(0x8c),p1(0x8d),p1(0x8e),p1(0x8f) ;\
+       .long   p1(0x90),p1(0x91),p1(0x92),p1(0x93),p1(0x94),p1(0x95),p1(0x96),p1(0x97) ;\
+    