diff -Nru linux-2.5.45.vanilla/drivers/char/eventpoll.c linux-2.5.45.epoll/drivers/char/eventpoll.c --- linux-2.5.45.vanilla/drivers/char/eventpoll.c Wed Oct 30 16:42:27 2002 +++ linux-2.5.45.epoll/drivers/char/eventpoll.c Sat Nov 2 18:20:16 2002 @@ -1,6 +1,6 @@ /* * drivers/char/eventpoll.c ( Efficent event polling implementation ) - * Copyright (C) 2001,...,2002 Davide Libenzi + * Copyright (C) 2001,...,2002 Davide Libenzi * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,19 +20,13 @@ #include #include #include -#include #include #include -#include -#include #include -#include #include #include #include #include -#include -#include #include #include #include @@ -63,19 +57,53 @@ #define DPI_SLAB_DEBUG 0 #endif /* #if DEBUG_DPI != 0 */ -#define INITIAL_HASH_BITS 7 -#define MAX_HASH_BITS 18 -#define RESIZE_LENGTH 2 +/* Maximum storage for the eventpoll interest set */ +#define EP_MAX_FDS_SIZE (1024 * 128) + +/* We don't want the hash to be smaller than this */ +#define EP_MIN_HASH_SIZE 101 +/* + * Event buffer dimension used to cache events before sending them in + * userspace with a __copy_to_user(). The event buffer is in stack, + * so keep this size fairly small. + */ +#define EP_EVENT_BUFF_SIZE 32 + +/* Maximum number of wait queue we can attach to */ +#define EP_MAX_POLL_QUEUE 2 + +/* Number of hash entries ( "struct list_head" ) inside a page */ +#define EP_HENTRY_X_PAGE (PAGE_SIZE / sizeof(struct list_head)) + +/* Maximum size of the hash in pages */ +#define EP_MAX_HPAGES (EP_MAX_FDS_SIZE / EP_HENTRY_X_PAGE + 1) + +/* Macro to allocate a "struct epitem" from the slab cache */ #define DPI_MEM_ALLOC() (struct epitem *) kmem_cache_alloc(dpi_cache, SLAB_KERNEL) + +/* Macro to free a "struct epitem" to the slab cache */ #define DPI_MEM_FREE(p) kmem_cache_free(dpi_cache, p) -#define IS_FILE_EPOLL(f) ((f)->f_op == &eventpoll_fops) +/* Fast test to see if the file is an evenpoll file */ +#define IS_FILE_EPOLL(f) ((f)->f_op == &eventpoll_fops) /* - * Type used for versioning events snapshots inside the double buffer. + * Remove the item from the list and perform its initialization. + * This is usefull for us because we can test if the item is linked + * using "EP_IS_LINKED(p)". */ -typedef unsigned long long event_version_t; +#define EP_LIST_DEL(p) do { list_del(p); INIT_LIST_HEAD(p); } while (0) + +/* Tells us if the item is currently linked */ +#define EP_IS_LINKED(p) (!list_empty(p)) + +/* Get the "struct epitem" from a wait queue pointer */ +#define EP_ITEM_FROM_WAIT(p) ((struct epitem *) container_of(p, struct eppoll_entry, wait)->base) + + + + /* * This structure is stored inside the "private_data" member of the file @@ -83,71 +111,44 @@ * interface. */ struct eventpoll { - /* - * Protect the evenpoll interface from sys_epoll_ctl(2), ioctl(EP_POLL) - * and ->write() concurrency. It basically serialize the add/remove/edit - * of items in the interest set. - */ - struct rw_semaphore acsem; + /* Used to link to the "struct eventpoll" list ( eplist ) */ + struct list_head llink; - /* - * Protect the this structure access. When the "acsem" is acquired - * togheter with this one, "acsem" should be acquired first. Or, - * "lock" nests inside "acsem". - */ + /* Protect the this structure access */ rwlock_t lock; - /* Wait queue used by sys_epoll_wait() and ioctl(EP_POLL) */ + /* Wait queue used by sys_epoll_wait() */ wait_queue_head_t wq; /* Wait queue used by file->poll() */ wait_queue_head_t poll_wait; - /* This is the hash used to store the "struct epitem" elements */ - struct list_head *hash; + /* List of ready file descriptors */ + struct list_head rdllist; - unsigned int hbits; - unsigned int hmask; - atomic_t hents; - atomic_t resize; + /* Size of the hash */ + int hsize; - /* Number of pages currently allocated in each side of the double buffer */ - int numpages; + /* Number of pages currently allocated for the hash */ + int nhpages; - /* - * Current page set pointer, switched from "pages0" and "pages1" each time - * ep_poll() returns events to the caller. - */ - char **pages; - - /* Each one of these contains the pages allocated for each side of - * the double buffer. - */ - char *pages0[MAX_EVENTPOLL_PAGES]; - char *pages1[MAX_EVENTPOLL_PAGES]; + /* Pages for the "struct epitem" hash */ + char *hpages[EP_MAX_HPAGES]; +}; - /* - * Variable containing the vma base address where the double buffer - * pages are mapped onto. - */ - unsigned long vmabase; +/* Wait structure used by the poll hooks */ +struct eppoll_entry { + /* The "base" pointer is set to the container "struct epitem" */ + void *base; /* - * Certain functions cannot be called if the double buffer pages are - * not allocated and if the memory mapping is not in place. This tells - * us that everything is setup to fully use the interface. + * Wait queue item that will be linked to the target file wait + * queue head. */ - atomic_t mmapped; - - /* Number of events currently available inside the current snapshot */ - int eventcnt; + wait_queue_t wait; - /* - * Variable storing the current "version" of the snapshot. It is used - * to validate the validity of the current slot pointed by the "index" - * member of a "struct epitem". - */ - event_version_t ver; + /* The wait queue head that linked the "wait" wait queue item */ + wait_queue_head_t *whead; }; /* @@ -158,6 +159,15 @@ /* List header used to link this structure to the eventpoll hash */ struct list_head llink; + /* List header used to link this structure to the eventpoll ready list */ + struct list_head rdllink; + + /* Number of active wait queue attached to poll operations */ + int nwait; + + /* Wait queue used to attach poll operations */ + struct eppoll_entry wait[EP_MAX_POLL_QUEUE]; + /* The "container" of this item */ struct eventpoll *ep; @@ -168,52 +178,48 @@ struct pollfd pfd; /* - * The index inside the current double buffer that stores the active - * event slot for this item ( file ). - */ - int index; - - /* - * The version that is used to validate if the current slot is still - * valid or if it refers to an old snapshot. It is matches togheter - * with the one inside the eventpoll structure. + * Used to keep track of the usage count of the structure. This avoids + * that the structure will desappear from underneath our processing. */ - event_version_t ver; + atomic_t usecnt; }; +static int ep_is_prime(int n); static int ep_getfd(int *efd, struct inode **einode, struct file **efile); static int ep_alloc_pages(char **pages, int numpages); static int ep_free_pages(char **pages, int numpages); -static int ep_init(struct eventpoll *ep); +static int ep_file_init(struct file *file, int hsize); +static int ep_hash_index(struct eventpoll *ep, struct file *file); +static struct list_head *ep_hash_entry(struct eventpoll *ep, int index); +static int ep_init(struct eventpoll *ep, int hsize); static void ep_free(struct eventpoll *ep); -static struct epitem *ep_find_nl(struct eventpoll *ep, int fd); -static struct epitem *ep_find(struct eventpoll *ep, int fd); -static int ep_hashresize(struct eventpoll *ep, unsigned long *kflags); -static int ep_insert(struct eventpoll *ep, struct pollfd *pfd); +static struct epitem *ep_find(struct eventpoll *ep, struct file *file); +static void ep_use_epitem(struct epitem *dpi); +static void ep_release_epitem(struct epitem *dpi); +static void ep_ptable_queue_proc(void *priv, wait_queue_head_t *whead); +static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfile); +static int ep_unlink(struct eventpoll *ep, struct epitem *dpi); static int ep_remove(struct eventpoll *ep, struct epitem *dpi); -static void notify_proc(struct file *file, void *data, unsigned long *local, - long *event); -static int open_eventpoll(struct inode *inode, struct file *file); -static int close_eventpoll(struct inode *inode, struct file *file); -static unsigned int poll_eventpoll(struct file *file, poll_table *wait); -static int write_eventpoll(struct file *file, const char *buffer, size_t count, - loff_t *ppos); -static int ep_poll(struct eventpoll *ep, struct evpoll *dvp); -static int ep_do_alloc_pages(struct eventpoll *ep, int numpages); -static int ioctl_eventpoll(struct inode *inode, struct file *file, - unsigned int cmd, unsigned long arg); -static void eventpoll_mm_open(struct vm_area_struct * vma); -static void eventpoll_mm_close(struct vm_area_struct * vma); -static int mmap_eventpoll(struct file *file, struct vm_area_struct *vma); +static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync); +static int ep_eventpoll_close(struct inode *inode, struct file *file); +static unsigned int ep_eventpoll_poll(struct file *file, poll_table *wait); +static int ep_events_transfer(struct eventpoll *ep, struct pollfd *events, int maxevents); +static int ep_poll(struct eventpoll *ep, struct pollfd *events, int maxevents, + int timeout); static int eventpollfs_delete_dentry(struct dentry *dentry); -static struct inode *get_eventpoll_inode(void); +static struct inode *ep_eventpoll_inode(void); static struct super_block *eventpollfs_get_sb(struct file_system_type *fs_type, int flags, char *dev_name, void *data); +/* Use to link togheter all the "struct eventpoll" */ +static struct list_head eplist; + +/* Serialize the access to "eplist" */ +static rwlock_t eplock; /* Slab cache used to allocate "struct epitem" */ static kmem_cache_t *dpi_cache; @@ -223,26 +229,8 @@ /* File callbacks that implement the eventpoll file behaviour */ static struct file_operations eventpoll_fops = { - .write = write_eventpoll, - .ioctl = ioctl_eventpoll, - .mmap = mmap_eventpoll, - .open = open_eventpoll, - .release = close_eventpoll, - .poll = poll_eventpoll -}; - -/* Memory mapping callbacks for the eventpoll file */ -static struct vm_operations_struct eventpoll_mmap_ops = { - .open = eventpoll_mm_open, - .close = eventpoll_mm_close, -}; - -/* - * The "struct miscdevice" is used to register the eventpoll device - * to make it suitable to be openend from a /dev file. - */ -static struct miscdevice eventpoll_miscdev = { - EVENTPOLL_MINOR, "eventpoll", &eventpoll_fops + .release = ep_eventpoll_close, + .poll = ep_eventpoll_poll }; /* @@ -262,27 +250,67 @@ +/* Report if the number is prime. Needed to correctly size the hash */ +static int ep_is_prime(int n) +{ + + if (n > 3) { + if (n & 1) { + int i, hn = n / 2; + + for (i = 3; i < hn; i += 2) + if (!(n % i)) + return 0; + } else + return 0; + } + return 1; +} + + +/* + * This is called from inside fs/file_table.c:__fput() to unlink files + * from the eventpoll interface. We need to have this facility to cleanup + * correctly files that are closed without being removed from the eventpoll + * interface. + */ +void ep_notify_file_close(struct file *file) +{ + unsigned long flags; + struct list_head *lnk; + struct eventpoll *ep; + struct epitem *dpi; + + read_lock_irqsave(&eplock, flags); + list_for_each(lnk, &eplist) { + ep = list_entry(lnk, struct eventpoll, llink); + + while ((dpi = ep_find(ep, file))) { + ep_remove(ep, dpi); + ep_release_epitem(dpi); + } + } + read_unlock_irqrestore(&eplock, flags); +} + + /* - * It opens an eventpoll file descriptor by allocating space for "maxfds" + * It opens an eventpoll file descriptor by suggesting a storage of "size" * file descriptors. It is the kernel part of the userspace epoll_create(2). */ -asmlinkage int sys_epoll_create(int maxfds) +asmlinkage int sys_epoll_create(int size) { - int error = -EINVAL, fd; - unsigned long addr; + int error, fd; struct inode *inode; struct file *file; - struct eventpoll *ep; DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_create(%d)\n", - current, maxfds)); + current, size)); - /* - * It is not possible to store more than MAX_FDS_IN_EVENTPOLL file - * descriptors inside the eventpoll interface. - */ - if (maxfds > MAX_FDS_IN_EVENTPOLL) - goto eexit_1; + /* Search the nearest prime number higher than "size" */ + for (; !ep_is_prime(size); size++); + if (size < EP_MIN_HASH_SIZE) + size = EP_MIN_HASH_SIZE; /* * Creates all the items needed to setup an eventpoll file. That is, @@ -292,39 +320,14 @@ if (error) goto eexit_1; - /* - * Calls the code to initialize the eventpoll file. This code is - * the same as the "open" file operation callback because inside - * ep_getfd() we did what the kernel usually does before invoking - * corresponding file "open" callback. - */ - error = open_eventpoll(inode, file); + /* Setup the file internal data structure ( "struct eventpoll" ) */ + error = ep_file_init(file, size); if (error) goto eexit_2; - /* The "private_data" member is setup by open_eventpoll() */ - ep = file->private_data; - - /* Alloc pages for the event double buffer */ - error = ep_do_alloc_pages(ep, EP_FDS_PAGES(maxfds + 1)); - if (error) - goto eexit_2; - - /* - * Create a user space mapping of the event double buffer to - * avoid kernel to user space memory copy when returning events - * to the caller. - */ - down_write(¤t->mm->mmap_sem); - addr = do_mmap_pgoff(file, 0, EP_MAP_SIZE(maxfds + 1), PROT_READ, - MAP_PRIVATE, 0); - up_write(¤t->mm->mmap_sem); - error = PTR_ERR((void *) addr); - if (IS_ERR((void *) addr)) - goto eexit_2; DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_create(%d) = %d\n", - current, maxfds, fd)); + current, size, fd)); return fd; @@ -332,7 +335,7 @@ sys_close(fd); eexit_1: DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_create(%d) = %d\n", - current, maxfds, error)); + current, size, error)); return error; } @@ -344,8 +347,8 @@ */ asmlinkage int sys_epoll_ctl(int epfd, int op, int fd, unsigned int events) { - int error = -EBADF; - struct file *file; + int error; + struct file *file, *tfile; struct eventpoll *ep; struct epitem *dpi; struct pollfd pfd; @@ -353,17 +356,29 @@ DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_ctl(%d, %d, %d, %u)\n", current, epfd, op, fd, events)); + /* Get the "struct file *" for the eventpoll file */ + error = -EBADF; file = fget(epfd); if (!file) goto eexit_1; + /* Get the "struct file *" for the target file */ + tfile = fget(fd); + if (!tfile) + goto eexit_2; + + /* The target file descriptor must support poll */ + error = -EPERM; + if (!tfile->f_op || !tfile->f_op->poll) + goto eexit_3; + /* * We have to check that the file structure underneath the file descriptor * the user passed to us _is_ an eventpoll file. */ error = -EINVAL; if (!IS_FILE_EPOLL(file)) - goto eexit_2; + goto eexit_3; /* * At this point it is safe to assume that the "private_data" contains @@ -371,20 +386,26 @@ */ ep = file->private_data; - down_write(&ep->acsem); - - pfd.fd = fd; - pfd.events = events | POLLERR | POLLHUP; - pfd.revents = 0; - - dpi = ep_find(ep, fd); + /* + * Try to lookup the file inside our hash table. When an item is found + * ep_find() increases the usage count of the item so that it won't + * desappear underneath us. The only thing that might happen, if someone + * tries very hard, is a double insertion of the same file descriptor. + * This does not rapresent a problem though and we don't really want + * to put an extra syncronization object to deal with this harmless condition. + */ + dpi = ep_find(ep, tfile); error = -EINVAL; switch (op) { case EP_CTL_ADD: - if (!dpi) - error = ep_insert(ep, &pfd); - else + if (!dpi) { + pfd.fd = fd; + pfd.events = events | POLLERR | POLLHUP; + pfd.revents = 0; + + error = ep_insert(ep, &pfd, tfile); + } else error = -EEXIST; break; case EP_CTL_DEL: @@ -402,11 +423,18 @@ break; } + /* + * The function ep_find() increments the usage count of the structure + * so, if this is not NULL, we need to release it. + */ + if (dpi) + ep_release_epitem(dpi); + DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_ctl(%d, %d, %d, %u) = %d\n", current, epfd, op, fd, events, error)); - up_write(&ep->acsem); - +eexit_3: + fput(tfile); eexit_2: fput(file); eexit_1: @@ -418,17 +446,22 @@ * Implement the event wait interface for the eventpoll file. It is the kernel * part of the user space epoll_wait(2). */ -asmlinkage int sys_epoll_wait(int epfd, struct pollfd const **events, int timeout) +asmlinkage int sys_epoll_wait(int epfd, struct pollfd *events, int maxevents, + int timeout) { - int error = -EBADF; - void *eaddr; + int error; struct file *file; struct eventpoll *ep; - struct evpoll dvp; - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d)\n", - current, epfd, events, timeout)); + DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d)\n", + current, epfd, events, maxevents, timeout)); + + /* Verify that the area passed by the user is writeable */ + if ((error = verify_area(VERIFY_WRITE, events, maxevents * sizeof(struct pollfd)))) + goto eexit_1; + /* Get the "struct file *" for the eventpoll file */ + error = -EBADF; file = fget(epfd); if (!file) goto eexit_1; @@ -447,26 +480,11 @@ */ ep = file->private_data; - /* - * It is possible that the user created an eventpoll file by open()ing - * the corresponding /dev/ file and he did not perform the correct - * initialization required by the old /dev/epoll interface. This test - * protect us from this scenario. - */ - error = -EINVAL; - if (!atomic_read(&ep->mmapped)) - goto eexit_2; - - dvp.ep_timeout = timeout; - error = ep_poll(ep, &dvp); - if (error > 0) { - eaddr = (void *) (ep->vmabase + dvp.ep_resoff); - if (copy_to_user(events, &eaddr, sizeof(struct pollfd *))) - error = -EFAULT; - } + /* Time to fish for events ... */ + error = ep_poll(ep, events, maxevents, timeout); - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d) = %d\n", - current, epfd, events, timeout, error)); + DNPRINTK(3, (KERN_INFO "[%p] eventpoll: sys_epoll_wait(%d, %p, %d, %d) = %d\n", + current, epfd, events, maxevents, timeout, error)); eexit_2: fput(file); @@ -494,7 +512,7 @@ goto eexit_1; /* Allocates an inode from the eventpoll file system */ - inode = get_eventpoll_inode(); + inode = ep_eventpoll_inode(); error = PTR_ERR(inode); if (IS_ERR(inode)) goto eexit_2; @@ -527,9 +545,9 @@ * with write() to add/remove/change interest sets. */ file->f_pos = 0; - file->f_flags = O_RDWR; + file->f_flags = O_RDONLY; file->f_op = &eventpoll_fops; - file->f_mode = FMODE_READ | FMODE_WRITE; + file->f_mode = FMODE_READ; file->f_version = 0; file->private_data = NULL; @@ -548,24 +566,24 @@ eexit_2: put_filp(file); eexit_1: - return error; + return error; } static int ep_alloc_pages(char **pages, int numpages) { - int ii; + int i; - for (ii = 0; ii < numpages; ii++) { - pages[ii] = (char *) __get_free_pages(GFP_KERNEL, 0); - if (!pages[ii]) { - for (--ii; ii >= 0; ii--) { - ClearPageReserved(virt_to_page(pages[ii])); - free_pages((unsigned long) pages[ii], 0); + for (i = 0; i < numpages; i++) { + pages[i] = (char *) __get_free_pages(GFP_KERNEL, 0); + if (!pages[i]) { + for (--i; i >= 0; i--) { + ClearPageReserved(virt_to_page(pages[i])); + free_pages((unsigned long) pages[i], 0); } return -ENOMEM; } - SetPageReserved(virt_to_page(pages[ii])); + SetPageReserved(virt_to_page(pages[i])); } return 0; } @@ -573,328 +591,364 @@ static int ep_free_pages(char **pages, int numpages) { - int ii; + int i; - for (ii = 0; ii < numpages; ii++) { - ClearPageReserved(virt_to_page(pages[ii])); - free_pages((unsigned long) pages[ii], 0); + for (i = 0; i < numpages; i++) { + ClearPageReserved(virt_to_page(pages[i])); + free_pages((unsigned long) pages[i], 0); } return 0; } -static int ep_init(struct eventpoll *ep) +static int ep_file_init(struct file *file, int hsize) +{ + int error; + unsigned long flags; + struct eventpoll *ep; + + if (!(ep = kmalloc(sizeof(struct eventpoll), GFP_KERNEL))) + return -ENOMEM; + + memset(ep, 0, sizeof(*ep)); + + error = ep_init(ep, hsize); + if (error) { + kfree(ep); + return error; + } + + file->private_data = ep; + + /* Add the structure to the linked list that links "struct eventpoll" */ + write_lock_irqsave(&eplock, flags); + list_add(&ep->llink, &eplist); + write_unlock_irqrestore(&eplock, flags); + + DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_file_init() ep=%p\n", + current, ep)); + return 0; +} + + +/* + * Calculate the index of the hash relative to "file". + */ +static int ep_hash_index(struct eventpoll *ep, struct file *file) +{ + + return (int) ((((unsigned long) file) / sizeof(struct file)) % ep->hsize); +} + + +/* + * Returns the hash entry ( struct list_head * ) of the passed index. + */ +static struct list_head *ep_hash_entry(struct eventpoll *ep, int index) +{ + + return (struct list_head *) (ep->hpages[index / EP_HENTRY_X_PAGE] + + (index % EP_HENTRY_X_PAGE) * sizeof(struct list_head)); +} + + +static int ep_init(struct eventpoll *ep, int hsize) { - int ii, hentries; + int error, i; - init_rwsem(&ep->acsem); + INIT_LIST_HEAD(&ep->llink); rwlock_init(&ep->lock); init_waitqueue_head(&ep->wq); init_waitqueue_head(&ep->poll_wait); - ep->hbits = INITIAL_HASH_BITS; - ep->hmask = (1 << ep->hbits) - 1; - atomic_set(&ep->hents, 0); - atomic_set(&ep->resize, 0); - atomic_set(&ep->mmapped, 0); - ep->numpages = 0; - ep->vmabase = 0; - ep->pages = ep->pages0; - ep->eventcnt = 0; - ep->ver = 1; + INIT_LIST_HEAD(&ep->rdllist); - hentries = ep->hmask + 1; - if (!(ep->hash = (struct list_head *) vmalloc(hentries * sizeof(struct list_head)))) - return -ENOMEM; + /* Hash allocation and setup */ + ep->hsize = hsize; + ep->nhpages = hsize / EP_HENTRY_X_PAGE + (hsize % EP_HENTRY_X_PAGE ? 1: 0); + error = ep_alloc_pages(ep->hpages, ep->nhpages); + if (error) + goto eexit_1; - for (ii = 0; ii < hentries; ii++) - INIT_LIST_HEAD(&ep->hash[ii]); + /* Initialize hash buckets */ + for (i = 0; i < ep->hsize; i++) + INIT_LIST_HEAD(ep_hash_entry(ep, i)); return 0; +eexit_1: + return error; } static void ep_free(struct eventpoll *ep) { - int ii; + int i; + unsigned long flags; struct list_head *lsthead; /* * Walks through the whole hash by unregistering file callbacks and * freeing each "struct epitem". */ - for (ii = 0; ii <= ep->hmask; ii++) { - lsthead = &ep->hash[ii]; + for (i = 0; i < ep->hsize; i++) { + lsthead = ep_hash_entry(ep, i); + + /* + * We need to lock this because we could be hit by + * ep_notify_file_close() while we're freeing this. + */ + write_lock_irqsave(&ep->lock, flags); + while (!list_empty(lsthead)) { struct epitem *dpi = list_entry(lsthead->next, struct epitem, llink); - file_notify_delcb(dpi->file, notify_proc); - list_del(lsthead->next); - DPI_MEM_FREE(dpi); + /* The function ep_unlink() must be called with held lock */ + ep_unlink(ep, dpi); + + /* We release the lock before releasing the "struct epitem" */ + write_unlock_irqrestore(&ep->lock, flags); + + ep_release_epitem(dpi); + + /* And then we reaquire the lock ... */ + write_lock_irqsave(&ep->lock, flags); } + + write_unlock_irqrestore(&ep->lock, flags); } - /* - * At this point we can free the hash and the pages used for the event - * double buffer. The ep_free() function is called from the "close" - * file operations callback, and this garanties us that the pages are - * already unmapped. - */ - vfree(ep->hash); - if (ep->numpages > 0) { - ep_free_pages(ep->pages0, ep->numpages); - ep_free_pages(ep->pages1, ep->numpages); - } + + /* Remove the structure to the linked list that links "struct eventpoll" */ + write_lock_irqsave(&eplock, flags); + EP_LIST_DEL(&ep->llink); + write_unlock_irqrestore(&eplock, flags); + + /* Free hash pages */ + if (ep->nhpages > 0) + ep_free_pages(ep->hpages, ep->nhpages); } /* - * No lock version of ep_find(), used when the code had to acquire the lock - * before calling the function. + * Search the file inside the eventpoll hash. It add usage count to + * the returned item, so the caller must call ep_release_epitem() + * after finished using the "struct epitem". */ -static struct epitem *ep_find_nl(struct eventpoll *ep, int fd) +static struct epitem *ep_find(struct eventpoll *ep, struct file *file) { - struct epitem *dpi = NULL; + unsigned long flags; struct list_head *lsthead, *lnk; + struct epitem *dpi = NULL; - lsthead = &ep->hash[fd & ep->hmask]; + read_lock_irqsave(&ep->lock, flags); + + lsthead = ep_hash_entry(ep, ep_hash_index(ep, file)); list_for_each(lnk, lsthead) { dpi = list_entry(lnk, struct epitem, llink); - if (dpi->pfd.fd == fd) break; + if (dpi->file == file) { + ep_use_epitem(dpi); + break; + } dpi = NULL; } - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_find(%d) -> %p\n", - current, fd, dpi)); + read_unlock_irqrestore(&ep->lock, flags); + + DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_find(%p) -> %p\n", + current, file, dpi)); return dpi; } -static struct epitem *ep_find(struct eventpoll *ep, int fd) +/* + * Increment the usage count of the "struct epitem" making it sure + * that the user will have a valid pointer to reference. + */ +static void ep_use_epitem(struct epitem *dpi) { - struct epitem *dpi; - unsigned long flags; - - read_lock_irqsave(&ep->lock, flags); - dpi = ep_find_nl(ep, fd); - - read_unlock_irqrestore(&ep->lock, flags); - - return dpi; + atomic_inc(&dpi->usecnt); } -static int ep_hashresize(struct eventpoll *ep, unsigned long *kflags) +/* + * Decrement ( release ) the usage count by signaling that the user + * has finished using the structure. It might lead to freeing the + * structure itself if the count goes to zero. + */ +static void ep_release_epitem(struct epitem *dpi) { - struct list_head *hash, *oldhash; - unsigned int hbits = ep->hbits + 1; - unsigned int hmask = (1 << hbits) - 1; - int ii, res, hentries = hmask + 1; - unsigned long flags = *kflags; - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_hashresize(%p) bits=%u\n", - current, ep, hbits)); - - write_unlock_irqrestore(&ep->lock, flags); - - res = -ENOMEM; - if (!(hash = (struct list_head *) vmalloc(hentries * sizeof(struct list_head)))) { - write_lock_irqsave(&ep->lock, flags); - goto eexit_1; - } - - for (ii = 0; ii < hentries; ii++) - INIT_LIST_HEAD(&hash[ii]); - - write_lock_irqsave(&ep->lock, flags); + if (atomic_dec_and_test(&dpi->usecnt)) + DPI_MEM_FREE(dpi); +} - oldhash = ep->hash; - for (ii = 0; ii <= ep->hmask; ii++) { - struct list_head *oldhead = &oldhash[ii], *lnk; - while (!list_empty(oldhead)) { - struct epitem *dpi = list_entry(lnk = oldhead->next, struct epitem, llink); +/* + * This is the callback that is used to add our wait queue to the + * target file wakeup lists. + */ +static void ep_ptable_queue_proc(void *priv, wait_queue_head_t *whead) +{ + struct epitem *dpi = priv; - list_del(lnk); - list_add(lnk, &hash[dpi->pfd.fd & hmask]); - } + /* No more than EP_MAX_POLL_QUEUE wait queue are supported */ + if (dpi->nwait < EP_MAX_POLL_QUEUE) { + add_wait_queue(whead, &dpi->wait[dpi->nwait].wait); + dpi->wait[dpi->nwait].whead = whead; + dpi->nwait++; } - - ep->hash = hash; - ep->hbits = hbits; - ep->hmask = hmask; - - write_unlock_irqrestore(&ep->lock, flags); - vfree(oldhash); - write_lock_irqsave(&ep->lock, flags); - - res = 0; -eexit_1: - *kflags = flags; - atomic_dec(&ep->resize); - return res; } -static int ep_insert(struct eventpoll *ep, struct pollfd *pfd) +static int ep_insert(struct eventpoll *ep, struct pollfd *pfd, struct file *tfile) { - int error; - struct epitem *dpi; - struct file *file; + int error, i, revents; unsigned long flags; - - if (atomic_read(&ep->hents) >= (ep->numpages * POLLFD_X_PAGE)) - return -E2BIG; - - file = fget(pfd->fd); - if (!file) - return -EBADF; + struct epitem *dpi; + poll_table pt; error = -ENOMEM; if (!(dpi = DPI_MEM_ALLOC())) goto eexit_1; + /* Item initialization follow here ... */ INIT_LIST_HEAD(&dpi->llink); + INIT_LIST_HEAD(&dpi->rdllink); dpi->ep = ep; - dpi->file = file; + dpi->file = tfile; dpi->pfd = *pfd; - dpi->index = -1; - dpi->ver = ep->ver - 1; + atomic_set(&dpi->usecnt, 1); + dpi->nwait = 0; + for (i = 0; i < EP_MAX_POLL_QUEUE; i++) { + init_waitqueue_func_entry(&dpi->wait[i].wait, ep_poll_callback); + dpi->wait[i].whead = NULL; + dpi->wait[i].base = dpi; + } + /* Attach the item to the poll hooks */ + poll_initwait_ex(&pt, 1, ep_ptable_queue_proc, dpi); + revents = tfile->f_op->poll(tfile, &pt); + poll_freewait(&pt); + + /* We have to drop the new item inside our item list to keep track of it */ write_lock_irqsave(&ep->lock, flags); - list_add(&dpi->llink, &ep->hash[pfd->fd & ep->hmask]); - atomic_inc(&ep->hents); + list_add(&dpi->llink, ep_hash_entry(ep, ep_hash_index(ep, tfile))); - if (!atomic_read(&ep->resize) && - (atomic_read(&ep->hents) >> ep->hbits) > RESIZE_LENGTH && - ep->hbits < MAX_HASH_BITS) { - atomic_inc(&ep->resize); - ep_hashresize(ep, &flags); - } + /* If the file is already "ready" we drop it inside the ready list */ + if (revents & pfd->events) + list_add(&dpi->rdllink, &ep->rdllist); write_unlock_irqrestore(&ep->lock, flags); - file_notify_addcb(file, notify_proc, dpi); - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_insert(%p, %d)\n", current, ep, pfd->fd)); - error = 0; -eexit_1: - fput(file); + return 0; +eexit_1: return error; } /* - * Removes a "struct epitem" from the eventpoll hash and deallocates - * all the associated resources. + * Unlink the "struct epitem" from all places it might have been hooked up. + * This function must be called with write IRQ lock on "ep->lock". */ -static int ep_remove(struct eventpoll *ep, struct epitem *dpi) +static int ep_unlink(struct eventpoll *ep, struct epitem *dpi) { - unsigned long flags; - struct pollfd *pfd, *lpfd; - struct epitem *ldpi; + int i; - /* First, removes the callback from the file callback list */ - file_notify_delcb(dpi->file, notify_proc); + /* + * It can happen that this one is called for an item already unlinked. + * The check protect us from doing a double unlink ( crash ). + */ + if (!EP_IS_LINKED(&dpi->llink)) + goto not_linked; - write_lock_irqsave(&ep->lock, flags); + /* + * At this point is safe to do the job, unlink the item from our list. + * This operation togheter with the above check closes the door to + * double unlinks. + */ + EP_LIST_DEL(&dpi->llink); - list_del(&dpi->llink); - atomic_dec(&ep->hents); + /* Removes poll wait queue hooks */ + for (i = 0; i < dpi->nwait; i++) + remove_wait_queue(dpi->wait[i].whead, &dpi->wait[i].wait); /* - * This is to remove stale events. We don't want that the removed file - * has a pending event that might be associated with a file inserted - * at a later time inside the eventpoll interface. this code checks - * if the currently removed file has a valid pending event and, if it does, - * manages things to remove it and decrement the currently available - * event count. + * If the item we are going to remove is inside the ready file descriptors + * we want to remove it from this list to avoid stale events. */ - if (dpi->index >= 0 && dpi->ver == ep->ver && dpi->index < ep->eventcnt) { - pfd = (struct pollfd *) (ep->pages[EVENT_PAGE_INDEX(dpi->index)] + - EVENT_PAGE_OFFSET(dpi->index)); - if (pfd->fd == dpi->pfd.fd && dpi->index < --ep->eventcnt) { - lpfd = (struct pollfd *) (ep->pages[EVENT_PAGE_INDEX(ep->eventcnt)] + - EVENT_PAGE_OFFSET(ep->eventcnt)); - *pfd = *lpfd; + if (EP_IS_LINKED(&dpi->rdllink)) + EP_LIST_DEL(&dpi->rdllink); - if ((ldpi = ep_find_nl(ep, pfd->fd))) ldpi->index = dpi->index; - } - } +not_linked: + + DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_unlink(%p, %d)\n", + current, ep, dpi->pfd.fd)); + + return 0; +} + + +/* + * Removes a "struct epitem" from the eventpoll hash and deallocates + * all the associated resources. + */ +static int ep_remove(struct eventpoll *ep, struct epitem *dpi) +{ + int error; + unsigned long flags; + + /* We need to acquire the write IRQ lock before calling ep_unlink() */ + write_lock_irqsave(&ep->lock, flags); + + /* Really unlink the item from the hash */ + error = ep_unlink(ep, dpi); write_unlock_irqrestore(&ep->lock, flags); + if (error) + goto eexit_1; + DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ep_remove(%p, %d)\n", current, ep, dpi->pfd.fd)); /* At this point it is safe to free the eventpoll item */ - DPI_MEM_FREE(dpi); + ep_release_epitem(dpi); - return 0; + error = 0; +eexit_1: + return error; } /* - * This is the event notify callback that is called from fs/fcblist.c because - * of the registration ( file_notify_addcb() ) done in ep_insert(). + * This is the callback that is passed to the wait queue wakeup + * machanism. It is called by the stored file descriptors when they + * have events to report. */ -static void notify_proc(struct file *file, void *data, unsigned long *local, - long *event) +static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync) { - struct epitem *dpi = data; + unsigned long flags; + struct epitem *dpi = EP_ITEM_FROM_WAIT(wait); struct eventpoll *ep = dpi->ep; - struct pollfd *pfd; - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: notify(%p, %p, %ld, %ld) ep=%p\n", - current, file, data, event[0], event[1], ep)); + DNPRINTK(3, (KERN_INFO "[%p] eventpoll: poll_callback(%p) dpi=%p ep=%p\n", + current, dpi->file, dpi, ep)); - /* - * We don't need to disable IRQs here because the callback dispatch - * routine inside fs/fcblist.c already call us with disabled IRQ. - */ - write_lock(&ep->lock); + write_lock_irqsave(&ep->lock, flags); - /* We're not expecting any of those events. Jump out soon ... */ - if (!(dpi->pfd.events & event[1])) - goto out; + /* If this file is already in the ready list we exit soon */ + if (EP_IS_LINKED(&dpi->rdllink)) + goto is_linked; - /* - * This logic determins if an active even slot is available for the - * currently signaled file, or if we have to make space for a new one - * and increment the number of ready file descriptors ( ep->eventcnt ). - */ - if (dpi->index < 0 || dpi->ver != ep->ver) { - if (ep->eventcnt >= (ep->numpages * POLLFD_X_PAGE)) - goto out; - dpi->index = ep->eventcnt++; - dpi->ver = ep->ver; - pfd = (struct pollfd *) (ep->pages[EVENT_PAGE_INDEX(dpi->index)] + - EVENT_PAGE_OFFSET(dpi->index)); - *pfd = dpi->pfd; - } else { - pfd = (struct pollfd *) (ep->pages[EVENT_PAGE_INDEX(dpi->index)] + - EVENT_PAGE_OFFSET(dpi->index)); - if (pfd->fd != dpi->pfd.fd) { - if (ep->eventcnt >= (ep->numpages * POLLFD_X_PAGE)) - goto out; - dpi->index = ep->eventcnt++; - pfd = (struct pollfd *) (ep->pages[EVENT_PAGE_INDEX(dpi->index)] + - EVENT_PAGE_OFFSET(dpi->index)); - *pfd = dpi->pfd; - } - } - - /* - * Merge event bits into the corresponding event slot inside the - * double buffer. - */ - pfd->revents |= (pfd->events & event[1]); + list_add(&dpi->rdllink, &ep->rdllist); +is_linked: /* * Wake up ( if active ) both the eventpoll wait list and the ->poll() * wait list. @@ -903,33 +957,13 @@ wake_up(&ep->wq); if (waitqueue_active(&ep->poll_wait)) wake_up(&ep->poll_wait); -out: - write_unlock(&ep->lock); -} - -static int open_eventpoll(struct inode *inode, struct file *file) -{ - int res; - struct eventpoll *ep; - - if (!(ep = kmalloc(sizeof(struct eventpoll), GFP_KERNEL))) - return -ENOMEM; - - memset(ep, 0, sizeof(*ep)); - if ((res = ep_init(ep))) { - kfree(ep); - return res; - } - - file->private_data = ep; - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: open() ep=%p\n", current, ep)); - return 0; + write_unlock_irqrestore(&ep->lock, flags); + return 1; } -static int close_eventpoll(struct inode *inode, struct file *file) +static int ep_eventpoll_close(struct inode *inode, struct file *file) { struct eventpoll *ep = file->private_data; @@ -943,107 +977,112 @@ } -static unsigned int poll_eventpoll(struct file *file, poll_table *wait) +static unsigned int ep_eventpoll_poll(struct file *file, poll_table *wait) { + unsigned int pollflags = 0; + unsigned long flags; struct eventpoll *ep = file->private_data; + /* Insert inside our poll wait queue */ poll_wait(file, &ep->poll_wait, wait); - if (ep->eventcnt) - return POLLIN | POLLRDNORM; - return 0; + /* Check our condition */ + read_lock_irqsave(&ep->lock, flags); + if (!list_empty(&ep->rdllist)) + pollflags = POLLIN | POLLRDNORM; + read_unlock_irqrestore(&ep->lock, flags); + + return pollflags; } -static int write_eventpoll(struct file *file, const char *buffer, size_t count, - loff_t *ppos) +/* + * Perform the transfer of events to user space. Optimize the copy by + * caching EP_EVENT_BUFF_SIZE events at a time and then copying it to user space. + */ +static int ep_events_transfer(struct eventpoll *ep, struct pollfd *events, int maxevents) { - int rcount; - struct eventpoll *ep = file->private_data; - struct epitem *dpi; - struct pollfd pfd; - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: write(%p, %d)\n", current, ep, count)); - - /* The size of the write must be a multiple of sizeof(struct pollfd) */ - rcount = -EINVAL; - if (count % sizeof(struct pollfd)) - goto eexit_1; + int eventcnt, ebufcnt, revents; + unsigned long flags; + struct list_head *lsthead = &ep->rdllist; + struct pollfd eventbuf[EP_EVENT_BUFF_SIZE]; + poll_table pt; /* - * And we have also to verify that that area is correctly accessible - * for the user. + * This is a special poll table initialization that will + * make poll_wait() to not perform any wait queue insertion when + * called by file->f_op->poll(). This is a fast way to retrieve + * file events with perform any queue insertion, hence saving CPU cycles. */ - if ((rcount = verify_area(VERIFY_READ, buffer, count))) - goto eexit_1; + poll_initwait_ex(&pt, 0, NULL, NULL); - down_write(&ep->acsem); + write_lock_irqsave(&ep->lock, flags); - rcount = 0; + for (eventcnt = 0, ebufcnt = 0; eventcnt < maxevents && !list_empty(lsthead);) { + struct epitem *dpi = list_entry(lsthead->next, struct epitem, rdllink); - while (count > 0) { - if (__copy_from_user(&pfd, buffer, sizeof(pfd))) { - rcount = -EFAULT; - goto eexit_2; - } + /* Remove the item from the ready list */ + EP_LIST_DEL(&dpi->rdllink); - dpi = ep_find(ep, pfd.fd); + /* Fetch event bits from the signaled file */ + revents = dpi->file->f_op->poll(dpi->file, &pt); - if (pfd.fd >= current->files->max_fds || !current->files->fd[pfd.fd]) - pfd.events = POLLREMOVE; - if (pfd.events & POLLREMOVE) { - if (dpi) { - ep_remove(ep, dpi); - rcount += sizeof(pfd); + if (revents & dpi->pfd.events) { + eventbuf[ebufcnt] = dpi->pfd; + eventbuf[ebufcnt].revents = revents & eventbuf[ebufcnt].events; + ebufcnt++; + + /* If our buffer page is full we need to flush it to user space */ + if (ebufcnt == EP_EVENT_BUFF_SIZE) { + /* + * We need to drop the irqlock before using the function + * __copy_to_user() because it might fault. + */ + write_unlock_irqrestore(&ep->lock, flags); + if (__copy_to_user(&events[eventcnt], eventbuf, + ebufcnt * sizeof(struct pollfd))) { + poll_freewait(&pt); + return -EFAULT; + } + eventcnt += ebufcnt; + ebufcnt = 0; + write_lock_irqsave(&ep->lock, flags); } } - else if (dpi) { - dpi->pfd.events = pfd.events; - rcount += sizeof(pfd); - } else { - pfd.revents = 0; - if (!ep_insert(ep, &pfd)) - rcount += sizeof(pfd); - } + } + write_unlock_irqrestore(&ep->lock, flags); - buffer += sizeof(pfd); - count -= sizeof(pfd); + /* There might be still something inside our event buffer */ + if (ebufcnt) { + if (__copy_to_user(&events[eventcnt], eventbuf, + ebufcnt * sizeof(struct pollfd))) + eventcnt = -EFAULT; + else + eventcnt += ebufcnt; } -eexit_2: - up_write(&ep->acsem); -eexit_1: - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: write(%p, %d) = %d\n", - current, ep, count, rcount)); + poll_freewait(&pt); - return rcount; + return eventcnt; } -static int ep_poll(struct eventpoll *ep, struct evpoll *dvp) +static int ep_poll(struct eventpoll *ep, struct pollfd *events, int maxevents, + int timeout) { - int res = 0; - long timeout; + int res = 0, eavail; unsigned long flags; + long jtimeout; wait_queue_t wait; - /* - * We don't want ep_poll() to be called if the correct sequence - * of operations are performed to initialize it. This won't happen - * for the system call interface but it could happen using the - * old /dev/epoll interface, that is maintained for compatibility. - */ - if (!atomic_read(&ep->mmapped)) - return -EINVAL; - write_lock_irqsave(&ep->lock, flags); res = 0; - if (!ep->eventcnt) { + if (list_empty(&ep->rdllist)) { /* * We don't have any available event to return to the caller. * We need to sleep here, and we will be wake up by - * notify_proc() when events will become available. + * ep_poll_callback() when events will become available. */ init_waitqueue_entry(&wait, current); add_wait_queue(&ep->wq, &wait); @@ -1053,17 +1092,17 @@ * and the overflow condition ( > MAX_SCHEDULE_TIMEOUT / HZ ). The * passed timeout is in milliseconds, that why (t * HZ) / 1000. */ - timeout = dvp->ep_timeout == -1 || dvp->ep_timeout > MAX_SCHEDULE_TIMEOUT / HZ ? - MAX_SCHEDULE_TIMEOUT: (dvp->ep_timeout * HZ) / 1000; + jtimeout = timeout == -1 || timeout > MAX_SCHEDULE_TIMEOUT / HZ ? + MAX_SCHEDULE_TIMEOUT: (timeout * HZ) / 1000; for (;;) { /* - * We don't want to sleep if the notify_proc() sends us + * We don't want to sleep if the ep_poll_callback() sends us * a wakeup in between. That's why we set the task state * to TASK_INTERRUPTIBLE before doing the checks. */ set_current_state(TASK_INTERRUPTIBLE); - if (ep->eventcnt || !timeout) + if (!list_empty(&ep->rdllist) || !jtimeout) break; if (signal_pending(current)) { res = -EINTR; @@ -1071,7 +1110,7 @@ } write_unlock_irqrestore(&ep->lock, flags); - timeout = schedule_timeout(timeout); + jtimeout = schedule_timeout(jtimeout); write_lock_irqsave(&ep->lock, flags); } remove_wait_queue(&ep->wq, &wait); @@ -1079,249 +1118,15 @@ set_current_state(TASK_RUNNING); } - /* - * If we've been wake up because of events became available, we need to: - * - * 1) null the number of available ready file descriptors - * 2) increment the version of the current ( next ) snapshot - * 3) swap the double buffer to return the current one to the caller - * 4) set the current ( for the user, previous for the interface ) offset - */ - if (!res && ep->eventcnt) { - res = ep->eventcnt; - ep->eventcnt = 0; - ++ep->ver; - if (ep->pages == ep->pages0) { - ep->pages = ep->pages1; - dvp->ep_resoff = 0; - } else { - ep->pages = ep->pages0; - dvp->ep_resoff = ep->numpages * PAGE_SIZE; - } - } - - write_unlock_irqrestore(&ep->lock, flags); - - return res; -} - - -static int ep_do_alloc_pages(struct eventpoll *ep, int numpages) -{ - int res, pgalloc, pgcpy; - unsigned long flags; - char **pages, **pages0, **pages1; - - if (atomic_read(&ep->mmapped)) - return -EBUSY; - if (numpages > MAX_EVENTPOLL_PAGES) - return -EINVAL; - - pgalloc = numpages - ep->numpages; - if ((pages = (char **) vmalloc(2 * (pgalloc + 1) * sizeof(char *))) == NULL) - return -ENOMEM; - pages0 = &pages[0]; - pages1 = &pages[pgalloc + 1]; - - if ((res = ep_alloc_pages(pages0, pgalloc))) - goto eexit_1; + /* Is it worth to try to dig for events ? */ + eavail = !list_empty(&ep->rdllist); - if ((res = ep_alloc_pages(pages1, pgalloc))) { - ep_free_pages(pages0, pgalloc); - goto eexit_1; - } - - write_lock_irqsave(&ep->lock, flags); - pgcpy = (ep->numpages + pgalloc) > numpages ? numpages - ep->numpages: pgalloc; - if (pgcpy > 0) { - memcpy(&ep->pages0[ep->numpages], pages0, pgcpy * sizeof(char *)); - memcpy(&ep->pages1[ep->numpages], pages1, pgcpy * sizeof(char *)); - ep->numpages += pgcpy; - } write_unlock_irqrestore(&ep->lock, flags); - if (pgcpy < pgalloc) { - if (pgcpy < 0) - pgcpy = 0; - ep_free_pages(&pages0[pgcpy], pgalloc - pgcpy); - ep_free_pages(&pages1[pgcpy], pgalloc - pgcpy); - } + /* Transfer events to user space */ + if (!res && eavail) + res = ep_events_transfer(ep, events, maxevents); -eexit_1: - vfree(pages); - return res; -} - - -static int ioctl_eventpoll(struct inode *inode, struct file *file, - unsigned int cmd, unsigned long arg) -{ - int res; - struct eventpoll *ep = file->private_data; - struct epitem *dpi; - unsigned long flags; - struct pollfd pfd; - struct evpoll dvp; - - switch (cmd) { - case EP_ALLOC: - res = ep_do_alloc_pages(ep, EP_FDS_PAGES(arg)); - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ioctl(%p, EP_ALLOC, %lu) == %d\n", - current, ep, arg, res)); - return res; - - case EP_FREE: - if (atomic_read(&ep->mmapped)) - return -EBUSY; - - res = -EINVAL; - write_lock_irqsave(&ep->lock, flags); - if (ep->numpages > 0) { - ep_free_pages(ep->pages0, ep->numpages); - ep_free_pages(ep->pages1, ep->numpages); - ep->numpages = 0; - ep->pages = ep->pages0; - res = 0; - } - write_unlock_irqrestore(&ep->lock, flags); - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ioctl(%p, EP_FREE) == %d\n", - current, ep, res)); - return res; - - case EP_POLL: - if (copy_from_user(&dvp, (void *) arg, sizeof(struct evpoll))) - return -EFAULT; - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ioctl(%p, EP_POLL, %d)\n", - current, ep, dvp.ep_timeout)); - - res = ep_poll(ep, &dvp); - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ioctl(%p, EP_POLL, %d) == %d\n", - current, ep, dvp.ep_timeout, res)); - - if (res > 0 && copy_to_user((void *) arg, &dvp, sizeof(struct evpoll))) - res = -EFAULT; - - return res; - - case EP_ISPOLLED: - if (copy_from_user(&pfd, (void *) arg, sizeof(struct pollfd))) - return 0; - - read_lock_irqsave(&ep->lock, flags); - - res = 0; - if (!(dpi = ep_find_nl(ep, pfd.fd))) - goto is_not_polled; - - pfd = dpi->pfd; - res = 1; - - is_not_polled: - read_unlock_irqrestore(&ep->lock, flags); - - if (res) - copy_to_user((void *) arg, &pfd, sizeof(struct pollfd)); - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: ioctl(%p, EP_ISPOLLED, %d) == %d\n", - current, ep, pfd.fd, res)); - return res; - } - - return -EINVAL; -} - - -static void eventpoll_mm_open(struct vm_area_struct * vma) -{ - struct file *file = vma->vm_file; - struct eventpoll *ep = file->private_data; - - if (ep) atomic_inc(&ep->mmapped); - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: mm_open(%p)\n", current, ep)); -} - - -static void eventpoll_mm_close(struct vm_area_struct * vma) -{ - struct file *file = vma->vm_file; - struct eventpoll *ep = file->private_data; - - if (ep && atomic_dec_and_test(&ep->mmapped)) - ep->vmabase = 0; - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: mm_close(%p)\n", current, ep)); -} - - -static int mmap_eventpoll(struct file *file, struct vm_area_struct *vma) -{ - struct eventpoll *ep = file->private_data; - unsigned long start; - int ii, res, numpages; - size_t mapsize; - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: mmap(%p, %lx, %lx)\n", - current, ep, vma->vm_start, vma->vm_pgoff << PAGE_SHIFT)); - - /* - * We need the eventpoll file to be RW but we don't want it to be - * mapped RW. This test perform the test and reject RW mmaping. - */ - if (vma->vm_flags & VM_WRITE) - return -EACCES; - - if ((vma->vm_pgoff << PAGE_SHIFT) != 0) - return -EINVAL; - - /* - * We need to verify that the mapped area covers all the allocated - * double buffer. - */ - mapsize = PAGE_ALIGN(vma->vm_end - vma->vm_start); - numpages = mapsize >> PAGE_SHIFT; - - res = -EINVAL; - if (numpages != (2 * ep->numpages)) - goto eexit_1; - - /* - * Map the double buffer starting from "vma->vm_start" up to - * "vma->vm_start + ep->numpages * PAGE_SIZE". - */ - start = vma->vm_start; - for (ii = 0; ii < ep->numpages; ii++) { - if ((res = remap_page_range(vma, start, __pa(ep->pages0[ii]), - PAGE_SIZE, vma->vm_page_prot))) - goto eexit_1; - start += PAGE_SIZE; - } - for (ii = 0; ii < ep->numpages; ii++) { - if ((res = remap_page_range(vma, start, __pa(ep->pages1[ii]), - PAGE_SIZE, vma->vm_page_prot))) - goto eexit_1; - start += PAGE_SIZE; - } - vma->vm_ops = &eventpoll_mmap_ops; - - /* Saves the base mapping address for later use in sys_epoll_wait(2) */ - ep->vmabase = vma->vm_start; - - /* - * Ok, mapping has been done. We can open the door to functions that - * requires the mapping to be in place. - */ - atomic_set(&ep->mmapped, 1); - - res = 0; -eexit_1: - - DNPRINTK(3, (KERN_INFO "[%p] eventpoll: mmap(%p, %lx, %lx) == %d\n", - current, ep, vma->vm_start, vma->vm_pgoff << PAGE_SHIFT, res)); return res; } @@ -1333,7 +1138,7 @@ } -static struct inode *get_eventpoll_inode(void) +static struct inode *ep_eventpoll_inode(void) { int error = -ENOMEM; struct inode *inode = new_inode(eventpoll_mnt->mnt_sb); @@ -1346,7 +1151,7 @@ /* * Mark the inode dirty from the very beginning, * that way it will never be moved to the dirty - * list because "mark_inode_dirty()" will think + * list because mark_inode_dirty() will think * that it already _is_ on the dirty list. */ inode->i_state = I_DIRTY; @@ -1374,11 +1179,17 @@ { int error; + /* Initialize the list that will link "struct eventpoll" */ + INIT_LIST_HEAD(&eplist); + + /* Initialize the rwlock used to access "eplist" */ + rwlock_init(&eplock); + /* Allocates slab cache used to allocate "struct epitem" items */ error = -ENOMEM; dpi_cache = kmem_cache_create("eventpoll", sizeof(struct epitem), - __alignof__(struct epitem), + 0, DPI_SLAB_DEBUG, NULL, NULL); if (!dpi_cache) goto eexit_1; @@ -1397,21 +1208,10 @@ if (IS_ERR(eventpoll_mnt)) goto eexit_3; - /* - * This is to maintain compatibility with the old /dev/epoll interface. - * We need to register a misc device so that the caller can open(2) it - * through a file inside /dev. - */ - error = misc_register(&eventpoll_miscdev); - if (error) - goto eexit_4; - printk(KERN_INFO "[%p] eventpoll: driver installed.\n", current); - return error; + return 0; -eexit_4: - mntput(eventpoll_mnt); eexit_3: unregister_filesystem(&eventpoll_fs_type); eexit_2: @@ -1427,7 +1227,6 @@ /* Undo all operations done inside eventpoll_init() */ unregister_filesystem(&eventpoll_fs_type); mntput(eventpoll_mnt); - misc_deregister(&eventpoll_miscdev); kmem_cache_destroy(dpi_cache); } diff -Nru linux-2.5.45.vanilla/fs/Makefile linux-2.5.45.epoll/fs/Makefile --- linux-2.5.45.vanilla/fs/Makefile Wed Oct 30 16:42:59 2002 +++ linux-2.5.45.epoll/fs/Makefile Fri Nov 1 12:06:33 2002 @@ -6,14 +6,14 @@ # export-objs := open.o dcache.o buffer.o bio.o inode.o dquot.o mpage.o aio.o \ - fcntl.o read_write.o dcookies.o fcblist.o + fcntl.o read_write.o dcookies.o obj-y := open.o read_write.o devices.o file_table.o buffer.o \ bio.o super.o block_dev.o char_dev.o stat.o exec.o pipe.o \ namei.o fcntl.o ioctl.o readdir.o select.o fifo.o locks.o \ dcache.o inode.o attr.o bad_inode.o file.o dnotify.o \ filesystems.o namespace.o seq_file.o xattr.o libfs.o \ - fs-writeback.o mpage.o direct-io.o aio.o fcblist.o + fs-writeback.o mpage.o direct-io.o aio.o ifneq ($(CONFIG_NFSD),n) ifneq ($(CONFIG_NFSD),) diff -Nru linux-2.5.45.vanilla/fs/fcblist.c linux-2.5.45.epoll/fs/fcblist.c --- linux-2.5.45.vanilla/fs/fcblist.c Wed Oct 30 16:43:07 2002 +++ linux-2.5.45.epoll/fs/fcblist.c Wed Dec 31 16:00:00 1969 @@ -1,146 +0,0 @@ -/* - * linux/fs/fcblist.c ( File event callbacks handling ) - * Copyright (C) 2001,...,2002 Davide Libenzi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * Davide Libenzi - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -long ion_band_table[NSIGPOLL] = { - ION_IN, /* POLL_IN */ - ION_OUT, /* POLL_OUT */ - ION_IN, /* POLL_MSG */ - ION_ERR, /* POLL_ERR */ - 0, /* POLL_PRI */ - ION_HUP /* POLL_HUP */ -}; - -long poll_band_table[NSIGPOLL] = { - POLLIN | POLLRDNORM, /* POLL_IN */ - POLLOUT | POLLWRNORM | POLLWRBAND, /* POLL_OUT */ - POLLIN | POLLRDNORM | POLLMSG, /* POLL_MSG */ - POLLERR, /* POLL_ERR */ - POLLPRI | POLLRDBAND, /* POLL_PRI */ - POLLHUP | POLLERR /* POLL_HUP */ -}; - - - -/* - * Walk through the file callback list by calling each registered callback - * with the event that happened on the "filep" file. Callbacks are called - * by holding a read lock on the callback list lock, and also by keeping - * local IRQs disabled. - */ -void file_notify_event(struct file *filep, long *event) -{ - unsigned long flags; - struct list_head *lnk, *lsthead; - - read_lock_irqsave(&filep->f_cblock, flags); - - lsthead = &filep->f_cblist; - list_for_each(lnk, lsthead) { - struct fcb_struct *fcbp = list_entry(lnk, struct fcb_struct, llink); - - fcbp->cbproc(filep, fcbp->data, fcbp->local, event); - } - - read_unlock_irqrestore(&filep->f_cblock, flags); -} - - -/* - * Add a new callback to the list of file callbacks. - */ -int file_notify_addcb(struct file *filep, - void (*cbproc)(struct file *, void *, unsigned long *, long *), - void *data) -{ - unsigned long flags; - struct fcb_struct *fcbp; - - if (!(fcbp = (struct fcb_struct *) kmalloc(sizeof(struct fcb_struct), GFP_KERNEL))) - return -ENOMEM; - - memset(fcbp, 0, sizeof(struct fcb_struct)); - fcbp->cbproc = cbproc; - fcbp->data = data; - - write_lock_irqsave(&filep->f_cblock, flags); - list_add_tail(&fcbp->llink, &filep->f_cblist); - write_unlock_irqrestore(&filep->f_cblock, flags); - - return 0; -} - - -/* - * Removes the callback "cbproc" from the file callback list. - */ -int file_notify_delcb(struct file *filep, - void (*cbproc)(struct file *, void *, unsigned long *, long *)) -{ - unsigned long flags; - struct list_head *lnk, *lsthead; - - write_lock_irqsave(&filep->f_cblock, flags); - - lsthead = &filep->f_cblist; - list_for_each(lnk, lsthead) { - struct fcb_struct *fcbp = list_entry(lnk, struct fcb_struct, llink); - - if (fcbp->cbproc == cbproc) { - list_del(lnk); - write_unlock_irqrestore(&filep->f_cblock, flags); - kfree(fcbp); - return 0; - } - } - - write_unlock_irqrestore(&filep->f_cblock, flags); - - return -ENOENT; -} - - -/* - * It is called at file cleanup time and removes all the registered callbacks. - */ -void file_notify_cleanup(struct file *filep) -{ - unsigned long flags; - struct list_head *lsthead; - - write_lock_irqsave(&filep->f_cblock, flags); - - lsthead = &filep->f_cblist; - while (!list_empty(lsthead)) { - struct fcb_struct *fcbp = list_entry(lsthead->next, struct fcb_struct, llink); - - list_del(lsthead->next); - write_unlock_irqrestore(&filep->f_cblock, flags); - kfree(fcbp); - write_lock_irqsave(&filep->f_cblock, flags); - } - - write_unlock_irqrestore(&filep->f_cblock, flags); -} - diff -Nru linux-2.5.45.vanilla/fs/file_table.c linux-2.5.45.epoll/fs/file_table.c --- linux-2.5.45.vanilla/fs/file_table.c Wed Oct 30 16:41:39 2002 +++ linux-2.5.45.epoll/fs/file_table.c Fri Nov 1 16:10:46 2002 @@ -8,12 +8,12 @@ #include #include #include -#include #include #include #include #include #include +#include /* sysctl tunables... */ struct files_stat_struct files_stat = {0, 0, NR_FILE}; @@ -59,7 +59,6 @@ f->f_gid = current->fsgid; f->f_owner.lock = RW_LOCK_UNLOCKED; list_add(&f->f_list, &anon_list); - file_notify_init(f); file_list_unlock(); return f; } @@ -104,7 +103,6 @@ filp->f_uid = current->fsuid; filp->f_gid = current->fsgid; filp->f_op = dentry->d_inode->i_fop; - file_notify_init(filp); if (filp->f_op->open) return filp->f_op->open(dentry->d_inode, filp); else @@ -126,7 +124,7 @@ struct vfsmount * mnt = file->f_vfsmnt; struct inode * inode = dentry->d_inode; - file_notify_cleanup(file); + ep_notify_file_close(file); locks_remove_flock(file); if (file->f_op && file->f_op->release) diff -Nru linux-2.5.45.vanilla/fs/pipe.c linux-2.5.45.epoll/fs/pipe.c --- linux-2.5.45.vanilla/fs/pipe.c Wed Oct 30 16:42:57 2002 +++ linux-2.5.45.epoll/fs/pipe.c Fri Nov 1 12:09:00 2002 @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -48,7 +47,7 @@ pipe_read(struct file *filp, char *buf, size_t count, loff_t *ppos) { struct inode *inode = filp->f_dentry->d_inode; - int do_wakeup, pfull; + int do_wakeup; ssize_t ret; /* pread is not allowed on pipes. */ @@ -64,7 +63,6 @@ down(PIPE_SEM(*inode)); for (;;) { int size = PIPE_LEN(*inode); - pfull = PIPE_FULL(*inode); if (size) { char *pipebuf = PIPE_BASE(*inode) + PIPE_START(*inode); ssize_t chars = PIPE_MAX_RCHUNK(*inode); @@ -110,18 +108,12 @@ if (!ret) ret = -ERESTARTSYS; break; } - /* Send notification message */ - if (pfull && !PIPE_FULL(*inode) && PIPE_WRITEFILE(*inode)) - file_send_notify(PIPE_WRITEFILE(*inode), ION_OUT, POLLOUT | POLLWRNORM | POLLWRBAND); if (do_wakeup) { wake_up_interruptible_sync(PIPE_WAIT(*inode)); kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT); } pipe_wait(inode); } - /* Send notification message */ - if (pfull && !PIPE_FULL(*inode) && PIPE_WRITEFILE(*inode)) - file_send_notify(PIPE_WRITEFILE(*inode), ION_OUT, POLLOUT | POLLWRNORM | POLLWRBAND); up(PIPE_SEM(*inode)); /* Signal writers asynchronously that there is more room. */ if (do_wakeup) { @@ -139,7 +131,7 @@ struct inode *inode = filp->f_dentry->d_inode; ssize_t ret; size_t min; - int do_wakeup, pempty; + int do_wakeup; /* pwrite is not allowed on pipes. */ if (unlikely(ppos != &filp->f_pos)) @@ -157,7 +149,6 @@ down(PIPE_SEM(*inode)); for (;;) { int free; - pempty = PIPE_EMPTY(*inode); if (!PIPE_READERS(*inode)) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; @@ -203,9 +194,6 @@ if (!ret) ret = -ERESTARTSYS; break; } - /* Send notification message */ - if (pempty && !PIPE_EMPTY(*inode) && PIPE_READFILE(*inode)) - file_send_notify(PIPE_READFILE(*inode), ION_IN, POLLIN | POLLRDNORM); if (do_wakeup) { wake_up_interruptible_sync(PIPE_WAIT(*inode)); kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN); @@ -215,9 +203,6 @@ pipe_wait(inode); PIPE_WAITING_WRITERS(*inode)--; } - /* Send notification message */ - if (pempty && !PIPE_EMPTY(*inode) && PIPE_READFILE(*inode)) - file_send_notify(PIPE_READFILE(*inode), ION_IN, POLLIN | POLLRDNORM); up(PIPE_SEM(*inode)); if (do_wakeup) { wake_up_interruptible(PIPE_WAIT(*inode)); @@ -281,22 +266,9 @@ static int pipe_release(struct inode *inode, int decr, int decw) { - struct file *rdfile, *wrfile; down(PIPE_SEM(*inode)); PIPE_READERS(*inode) -= decr; PIPE_WRITERS(*inode) -= decw; - rdfile = PIPE_READFILE(*inode); - wrfile = PIPE_WRITEFILE(*inode); - if (decr && !PIPE_READERS(*inode)) { - PIPE_READFILE(*inode) = NULL; - if (wrfile) - file_send_notify(wrfile, ION_HUP, POLLHUP); - } - if (decw && !PIPE_WRITERS(*inode)) { - PIPE_WRITEFILE(*inode) = NULL; - if (rdfile) - file_send_notify(rdfile, ION_HUP, POLLHUP); - } if (!PIPE_READERS(*inode) && !PIPE_WRITERS(*inode)) { struct pipe_inode_info *info = inode->i_pipe; inode->i_pipe = NULL; @@ -516,7 +488,6 @@ PIPE_READERS(*inode) = PIPE_WRITERS(*inode) = 0; PIPE_WAITING_WRITERS(*inode) = 0; PIPE_RCOUNTER(*inode) = PIPE_WCOUNTER(*inode) = 1; - PIPE_READFILE(*inode) = PIPE_WRITEFILE(*inode) = NULL; *PIPE_FASYNC_READERS(*inode) = *PIPE_FASYNC_WRITERS(*inode) = NULL; return inode; @@ -624,9 +595,6 @@ f2->f_op = &write_pipe_fops; f2->f_mode = 2; f2->f_version = 0; - - PIPE_READFILE(*inode) = f1; - PIPE_WRITEFILE(*inode) = f2; fd_install(i, f1); fd_install(j, f2); diff -Nru linux-2.5.45.vanilla/fs/select.c linux-2.5.45.epoll/fs/select.c --- linux-2.5.45.vanilla/fs/select.c Wed Oct 30 16:41:37 2002 +++ linux-2.5.45.epoll/fs/select.c Fri Nov 1 14:15:44 2002 @@ -77,6 +77,14 @@ { struct poll_table_page *table = p->table; + if (!p->queue) + return; + + if (p->qproc) { + p->qproc(p->priv, wait_address); + return; + } + if (!table || POLL_TABLE_FULL(table)) { struct poll_table_page *new_table; diff -Nru linux-2.5.45.vanilla/include/linux/eventpoll.h linux-2.5.45.epoll/include/linux/eventpoll.h --- linux-2.5.45.vanilla/include/linux/eventpoll.h Wed Oct 30 16:42:54 2002 +++ linux-2.5.45.epoll/include/linux/eventpoll.h Sat Nov 2 17:38:49 2002 @@ -1,6 +1,6 @@ /* * include/linux/eventpoll.h ( Efficent event polling implementation ) - * Copyright (C) 2001,...,2002 Davide Libenzi + * Copyright (C) 2001,...,2002 Davide Libenzi * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,37 +14,25 @@ #ifndef _LINUX_EVENTPOLL_H #define _LINUX_EVENTPOLL_H +/* Forward declarations to avoid compiler errors */ +struct file; +struct pollfd; -#define EVENTPOLL_MINOR 124 -#define POLLFD_X_PAGE (PAGE_SIZE / sizeof(struct pollfd)) -#define MAX_FDS_IN_EVENTPOLL (1024 * 128) -#define MAX_EVENTPOLL_PAGES (MAX_FDS_IN_EVENTPOLL / POLLFD_X_PAGE) -#define EVENT_PAGE_INDEX(n) ((n) / POLLFD_X_PAGE) -#define EVENT_PAGE_REM(n) ((n) % POLLFD_X_PAGE) -#define EVENT_PAGE_OFFSET(n) (((n) % POLLFD_X_PAGE) * sizeof(struct pollfd)) -#define EP_FDS_PAGES(n) (((n) + POLLFD_X_PAGE - 1) / POLLFD_X_PAGE) -#define EP_MAP_SIZE(n) (EP_FDS_PAGES(n) * PAGE_SIZE * 2) - - -struct evpoll { - int ep_timeout; - unsigned long ep_resoff; -}; - -#define EP_ALLOC _IOR('P', 1, int) -#define EP_POLL _IOWR('P', 2, struct evpoll) -#define EP_FREE _IO('P', 3) -#define EP_ISPOLLED _IOWR('P', 4, struct pollfd) +/* Valid opcodes to issue to sys_epoll_ctl() */ #define EP_CTL_ADD 1 #define EP_CTL_DEL 2 #define EP_CTL_MOD 3 -asmlinkage int sys_epoll_create(int maxfds); +/* Kernel space functions implementing the user space "epoll" API */ +asmlinkage int sys_epoll_create(int size); asmlinkage int sys_epoll_ctl(int epfd, int op, int fd, unsigned int events); -asmlinkage int sys_epoll_wait(int epfd, struct pollfd const **events, int timeout); +asmlinkage int sys_epoll_wait(int epfd, struct pollfd *events, int maxevents, + int timeout); +/* Used in fs/file_table.c:__fput() to unlink files from the eventpoll interface */ +void ep_notify_file_close(struct file *file); #endif diff -Nru linux-2.5.45.vanilla/include/linux/fcblist.h linux-2.5.45.epoll/include/linux/fcblist.h --- linux-2.5.45.vanilla/include/linux/fcblist.h Wed Oct 30 16:43:34 2002 +++ linux-2.5.45.epoll/include/linux/fcblist.h Wed Dec 31 16:00:00 1969 @@ -1,71 +0,0 @@ -/* - * include/linux/fcblist.h ( File event callbacks handling ) - * Copyright (C) 2001,...,2002 Davide Libenzi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * Davide Libenzi - * - */ - -#ifndef __LINUX_FCBLIST_H -#define __LINUX_FCBLIST_H - -#include -#include -#include -#include -#include - - - -/* file callback notification events */ -#define ION_IN 1 -#define ION_OUT 2 -#define ION_HUP 3 -#define ION_ERR 4 - -#define FCB_LOCAL_SIZE 4 - - -struct fcb_struct { - struct list_head llink; - void (*cbproc)(struct file *, void *, unsigned long *, long *); - void *data; - unsigned long local[FCB_LOCAL_SIZE]; -}; - - -extern long ion_band_table[]; -extern long poll_band_table[]; - - -void file_notify_event(struct file *filep, long *event); - -int file_notify_addcb(struct file *filep, - void (*cbproc)(struct file *, void *, unsigned long *, long *), - void *data); - -int file_notify_delcb(struct file *filep, - void (*cbproc)(struct file *, void *, unsigned long *, long *)); - -void file_notify_cleanup(struct file *filep); - - -static inline void file_notify_init(struct file *filep) -{ - rwlock_init(&filep->f_cblock); - INIT_LIST_HEAD(&filep->f_cblist); -} - -static inline void file_send_notify(struct file *filep, long ioevt, long plevt) -{ - long event[] = { ioevt, plevt, -1 }; - - file_notify_event(filep, event); -} - -#endif diff -Nru linux-2.5.45.vanilla/include/linux/fs.h linux-2.5.45.epoll/include/linux/fs.h --- linux-2.5.45.vanilla/include/linux/fs.h Wed Oct 30 16:42:21 2002 +++ linux-2.5.45.epoll/include/linux/fs.h Fri Nov 1 12:10:05 2002 @@ -504,10 +504,6 @@ /* needed for tty driver, and maybe others */ void *private_data; - - /* file callback list */ - rwlock_t f_cblock; - struct list_head f_cblist; }; extern spinlock_t files_lock; #define file_list_lock() spin_lock(&files_lock); diff -Nru linux-2.5.45.vanilla/include/linux/pipe_fs_i.h linux-2.5.45.epoll/include/linux/pipe_fs_i.h --- linux-2.5.45.vanilla/include/linux/pipe_fs_i.h Wed Oct 30 16:43:07 2002 +++ linux-2.5.45.epoll/include/linux/pipe_fs_i.h Fri Nov 1 12:10:38 2002 @@ -12,8 +12,6 @@ unsigned int waiting_writers; unsigned int r_counter; unsigned int w_counter; - struct file *rdfile; - struct file *wrfile; struct fasync_struct *fasync_readers; struct fasync_struct *fasync_writers; }; @@ -32,8 +30,6 @@ #define PIPE_WAITING_WRITERS(inode) ((inode).i_pipe->waiting_writers) #define PIPE_RCOUNTER(inode) ((inode).i_pipe->r_counter) #define PIPE_WCOUNTER(inode) ((inode).i_pipe->w_counter) -#define PIPE_READFILE(inode) ((inode).i_pipe->rdfile) -#define PIPE_WRITEFILE(inode) ((inode).i_pipe->wrfile) #define PIPE_FASYNC_READERS(inode) (&((inode).i_pipe->fasync_readers)) #define PIPE_FASYNC_WRITERS(inode) (&((inode).i_pipe->fasync_writers)) diff -Nru linux-2.5.45.vanilla/include/linux/poll.h linux-2.5.45.epoll/include/linux/poll.h --- linux-2.5.45.vanilla/include/linux/poll.h Wed Oct 30 16:42:54 2002 +++ linux-2.5.45.epoll/include/linux/poll.h Fri Nov 1 14:28:06 2002 @@ -13,6 +13,9 @@ struct poll_table_page; typedef struct poll_table_struct { + int queue; + void *priv; + void (*qproc)(void *, wait_queue_head_t *); int error; struct poll_table_page * table; } poll_table; @@ -27,9 +30,24 @@ static inline void poll_initwait(poll_table* pt) { + pt->queue = 1; + pt->qproc = NULL; + pt->priv = NULL; pt->error = 0; pt->table = NULL; } + +static inline void poll_initwait_ex(poll_table* pt, int queue, + void (*qproc)(void *, wait_queue_head_t *), + void *priv) +{ + pt->queue = queue; + pt->qproc = qproc; + pt->priv = priv; + pt->error = 0; + pt->table = NULL; +} + extern void poll_freewait(poll_table* pt); diff -Nru linux-2.5.45.vanilla/include/net/sock.h linux-2.5.45.epoll/include/net/sock.h --- linux-2.5.45.vanilla/include/net/sock.h Wed Oct 30 16:43:33 2002 +++ linux-2.5.45.epoll/include/net/sock.h Fri Nov 1 12:11:53 2002 @@ -52,9 +52,6 @@ #include #include #include -#include -#include -#include /* * This structure really needs to be cleaned up. @@ -769,13 +766,8 @@ static inline void sk_wake_async(struct sock *sk, int how, int band) { - if (sk->socket) { - if (sk->socket->file) - file_send_notify(sk->socket->file, ion_band_table[band - POLL_IN], - poll_band_table[band - POLL_IN]); - if (sk->socket->fasync_list) - sock_wake_async(sk->socket, how, band); - } + if (sk->socket && sk->socket->fasync_list) + sock_wake_async(sk->socket, how, band); } #define SOCK_MIN_SNDBUF 2048 diff -Nru linux-2.5.45.vanilla/net/ipv4/tcp.c linux-2.5.45.epoll/net/ipv4/tcp.c --- linux-2.5.45.vanilla/net/ipv4/tcp.c Wed Oct 30 16:42:21 2002 +++ linux-2.5.45.epoll/net/ipv4/tcp.c Fri Nov 1 12:15:17 2002 @@ -476,8 +476,8 @@ if (sk->sleep && waitqueue_active(sk->sleep)) wake_up_interruptible(sk->sleep); - if (!(sk->shutdown & SEND_SHUTDOWN)) - sk_wake_async(sk, 2, POLL_OUT); + if (sock->fasync_list && !(sk->shutdown & SEND_SHUTDOWN)) + sock_wake_async(sock, 2, POLL_OUT); } }