diff options
Diffstat (limited to 'drivers/firewire/core-cdev.c')
| -rw-r--r-- | drivers/firewire/core-cdev.c | 301 |
1 files changed, 151 insertions, 150 deletions
diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c index 664952a67a11..87efd65e6b00 100644 --- a/drivers/firewire/core-cdev.c +++ b/drivers/firewire/core-cdev.c @@ -47,14 +47,14 @@ #define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW 5 #define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP 6 -static DEFINE_SPINLOCK(phy_receiver_list_lock); +static DEFINE_MUTEX(phy_receiver_list_mutex); static LIST_HEAD(phy_receiver_list); struct client { u32 version; struct fw_device *device; - spinlock_t lock; + struct mutex mutex; bool in_shutdown; struct xarray resource_xa; struct list_head event_list; @@ -133,16 +133,17 @@ struct iso_resource_params { s32 bandwidth; }; +enum { + ISO_RES_AUTO_ALLOC, + ISO_RES_AUTO_REALLOC, + ISO_RES_AUTO_DEALLOC, +}; + struct iso_resource_auto { struct client_resource resource; struct client *client; - /* Schedule work and access todo only with client->lock held. */ struct delayed_work work; - enum { - ISO_RES_AUTO_ALLOC, - ISO_RES_AUTO_REALLOC, - ISO_RES_AUTO_DEALLOC, - } todo; + atomic_t todo; // one of ISO_RES_AUTO_XXX. int generation; struct iso_resource_params params; struct iso_resource_event *e_alloc, *e_dealloc; @@ -196,9 +197,7 @@ static int is_outbound_transaction_resource(const struct client_resource *resour static void schedule_iso_resource_auto(struct iso_resource_auto *r, unsigned long delay) { - client_get(r->client); - if (!queue_delayed_work(fw_workqueue, &r->work, delay)) - client_put(r->client); + queue_delayed_work(fw_workqueue, &r->work, delay); } /* @@ -316,7 +315,7 @@ static int fw_device_op_open(struct inode *inode, struct file *file) } client->device = device; - spin_lock_init(&client->lock); + mutex_init(&client->mutex); xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH); INIT_LIST_HEAD(&client->event_list); init_waitqueue_head(&client->wait); @@ -339,50 +338,61 @@ static void queue_event(struct client *client, struct event *event, event->v[1].data = data1; event->v[1].size = size1; - scoped_guard(spinlock_irqsave, &client->lock) { - if (client->in_shutdown) + scoped_guard(mutex, &client->mutex) { + if (client->in_shutdown) { kfree(event); - else + } else { list_add_tail(&event->link, &client->event_list); - } - wake_up_interruptible(&client->wait); + wake_up_interruptible(&client->wait); + } + } } -static int dequeue_event(struct client *client, - char __user *buffer, size_t count) +static ssize_t dequeue_event(struct client *client, char __user *buffer, size_t count) { struct event *event; - size_t size, total; - int i, ret; - ret = wait_event_interruptible(client->wait, - !list_empty(&client->event_list) || - fw_device_is_shutdown(client->device)); - if (ret < 0) - return ret; + // After the following block, the event pointer above is guaranteed to have a correct value. + { + mutex_lock(&client->mutex); + + // This could be replaced with wait_var_event_any_lock() if poll_wait() alternative + // would be introduced. + int ret = ___wait_event(client->wait, + !list_empty(&client->event_list) || fw_device_is_shutdown(client->device), + TASK_INTERRUPTIBLE, 0, 0, + mutex_unlock(&client->mutex); + schedule(); + mutex_lock(&client->mutex) + ); + if (ret < 0) { + mutex_unlock(&client->mutex); + return ret; + } - if (list_empty(&client->event_list) && - fw_device_is_shutdown(client->device)) - return -ENODEV; + if (fw_device_is_shutdown(client->device)) { + mutex_unlock(&client->mutex); + return -ENODEV; + } - scoped_guard(spinlock_irq, &client->lock) { event = list_first_entry(&client->event_list, struct event, link); list_del(&event->link); + + mutex_unlock(&client->mutex); } - total = 0; - for (i = 0; i < ARRAY_SIZE(event->v) && total < count; i++) { - size = min(event->v[i].size, count - total); - if (copy_to_user(buffer + total, event->v[i].data, size)) { + ssize_t ret = 0; + + for (int i = 0; i < ARRAY_SIZE(event->v) && ret < count; i++) { + size_t size = min(event->v[i].size, count - ret); + if (copy_to_user(buffer + ret, event->v[i].data, size)) { ret = -EFAULT; - goto out; + break; } - total += size; + ret += size; } - ret = total; - out: kfree(event); return ret; @@ -398,19 +408,27 @@ static ssize_t fw_device_op_read(struct file *file, char __user *buffer, static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event, struct client *client) +__must_hold(&client->device->client_list_mutex) { - struct fw_card *card = client->device->card; - - guard(spinlock_irq)(&card->lock); + lockdep_assert_held(&client->device->client_list_mutex); + // This member is related to the above mutex. In detail, see 93b37905f70 ("firewire: cdev: + // prevent race between first get_info ioctl and bus reset event queuing"). event->closure = client->bus_reset_closure; event->type = FW_CDEV_EVENT_BUS_RESET; + event->generation = client->device->generation; + smp_rmb(); event->node_id = client->device->node_id; - event->local_node_id = card->local_node->node_id; - event->bm_node_id = card->bm_node_id; - event->irm_node_id = card->irm_node->node_id; - event->root_node_id = card->root_node->node_id; + + struct fw_card *card = client->device->card; + + scoped_guard(spinlock_irq, &card->lock) { + event->local_node_id = card->local_node->node_id; + event->bm_node_id = card->bm_node_id; + event->irm_node_id = card->irm_node->node_id; + event->root_node_id = card->root_node->node_id; + } } static void for_each_client(struct fw_device *device, @@ -439,11 +457,11 @@ static void queue_bus_reset_event(struct client *client) queue_event(client, &e->event, &e->reset, sizeof(e->reset), NULL, 0); - guard(spinlock_irq)(&client->lock); - - xa_for_each(&client->resource_xa, index, resource) { - if (is_iso_resource_auto(resource)) - schedule_iso_resource_auto(to_iso_resource_auto(resource), 0); + scoped_guard(mutex, &client->mutex) { + xa_for_each(&client->resource_xa, index, resource) { + if (is_iso_resource_auto(resource)) + schedule_iso_resource_auto(to_iso_resource_auto(resource), 0); + } } } @@ -488,8 +506,6 @@ union ioctl_arg { static int ioctl_get_info(struct client *client, union ioctl_arg *arg) { struct fw_cdev_get_info *a = &arg->get_info; - struct fw_cdev_event_bus_reset bus_reset; - unsigned long ret = 0; client->version = a->version; a->version = FW_CDEV_KERNEL_VERSION; @@ -497,52 +513,58 @@ static int ioctl_get_info(struct client *client, union ioctl_arg *arg) scoped_guard(rwsem_read, &fw_device_rwsem) { if (a->rom != 0) { - size_t want = a->rom_length; - size_t have = client->device->config_rom_length * 4; + size_t length = min_t(size_t, a->rom_length, + client->device->config_rom_length * 4); - ret = copy_to_user(u64_to_uptr(a->rom), client->device->config_rom, - min(want, have)); - if (ret != 0) + if (copy_to_user(u64_to_uptr(a->rom), client->device->config_rom, length)) return -EFAULT; } a->rom_length = client->device->config_rom_length * 4; } - guard(mutex)(&client->device->client_list_mutex); + scoped_guard(mutex, &client->device->client_list_mutex) { + // Coordinate concurrent access to this member with bus reset event handling, see + // 93b37905f70 ("firewire: cdev: prevent race between first get_info ioctl and bus + // reset event queuing"). + client->bus_reset_closure = a->bus_reset_closure; + + if (a->bus_reset != 0) { + struct fw_cdev_event_bus_reset bus_reset; + + memset(&bus_reset, 0, sizeof(bus_reset)); + fill_bus_reset_event(&bus_reset, client); + + // This structure has 4 bytes of trailing padding under the System V ABI + // on most architectures (due to 8-byte alignment of the long long type), + // except for Intel386 (where long long type is aligned to 4 bytes). In + // either case, the effective length is 36 bytes. + if (copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36)) + return -EFAULT; + } - client->bus_reset_closure = a->bus_reset_closure; - if (a->bus_reset != 0) { - fill_bus_reset_event(&bus_reset, client); - /* unaligned size of bus_reset is 36 bytes */ - ret = copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36); + if (list_empty(&client->link)) + list_add_tail(&client->link, &client->device->client_list); } - if (ret == 0 && list_empty(&client->link)) - list_add_tail(&client->link, &client->device->client_list); - return ret ? -EFAULT : 0; + return 0; } static int add_client_resource(struct client *client, struct client_resource *resource, - gfp_t gfp_mask) + client_resource_release_fn_t release) { - scoped_guard(spinlock_irqsave, &client->lock) { + scoped_guard(mutex, &client->mutex) { u32 index; int ret; if (client->in_shutdown) return -ECANCELED; - if (gfpflags_allow_blocking(gfp_mask)) { - ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b, - GFP_NOWAIT); - } else { - ret = xa_alloc_bh(&client->resource_xa, &index, resource, - xa_limit_32b, GFP_NOWAIT); - } + ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b, GFP_KERNEL); if (ret < 0) return ret; resource->handle = index; + resource->release = release; client_get(client); } @@ -556,7 +578,7 @@ static int release_client_resource(struct client *client, u32 handle, unsigned long index = handle; struct client_resource *resource; - scoped_guard(spinlock_irq, &client->lock) { + scoped_guard(mutex, &client->mutex) { if (client->in_shutdown) return -EINVAL; @@ -589,7 +611,7 @@ static void complete_transaction(struct fw_card *card, int rcode, u32 request_ts struct client *client = e->client; unsigned long index = e->r.resource.handle; - scoped_guard(spinlock_irqsave, &client->lock) { + scoped_guard(mutex, &client->mutex) { xa_erase(&client->resource_xa, index); if (client->in_shutdown) wake_up(&client->tx_flush_wait); @@ -661,8 +683,8 @@ static int init_request(struct client *client, request->length < 4) return -EINVAL; - e = kmalloc(sizeof(*e) + request->length, GFP_KERNEL); - if (e == NULL) + e = kzalloc_flex(*e, rsp.with_tstamp.data, DIV_ROUND_UP(request->length, sizeof(u32))); + if (!e) return -ENOMEM; e->client = client; @@ -687,8 +709,7 @@ static int init_request(struct client *client, goto failed; } - e->r.resource.release = release_transaction; - ret = add_client_resource(client, &e->r.resource, GFP_KERNEL); + ret = add_client_resource(client, &e->r.resource, release_transaction); if (ret < 0) goto failed; @@ -760,8 +781,8 @@ static void handle_request(struct fw_card *card, struct fw_request *request, if (is_fcp) fw_request_get(request); - r = kmalloc_obj(*r, GFP_ATOMIC); - e = kmalloc_obj(*e, GFP_ATOMIC); + r = kmalloc_obj(*r); + e = kmalloc_obj(*e); if (r == NULL || e == NULL) goto failed; @@ -771,8 +792,7 @@ static void handle_request(struct fw_card *card, struct fw_request *request, r->data = payload; r->length = length; - r->resource.release = release_request; - ret = add_client_resource(handler->client, &r->resource, GFP_ATOMIC); + ret = add_client_resource(handler->client, &r->resource, release_request); if (ret < 0) goto failed; @@ -875,8 +895,7 @@ static int ioctl_allocate(struct client *client, union ioctl_arg *arg) } a->offset = r->handler.offset; - r->resource.release = release_address_handler; - ret = add_client_resource(client, &r->resource, GFP_KERNEL); + ret = add_client_resource(client, &r->resource, release_address_handler); if (ret < 0) { release_address_handler(client, &r->resource); return ret; @@ -975,8 +994,7 @@ static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg) if (ret < 0) goto failed; - r->resource.release = release_descriptor; - ret = add_client_resource(client, &r->resource, GFP_KERNEL); + ret = add_client_resource(client, &r->resource, release_descriptor); if (ret < 0) { fw_core_remove_descriptor(&r->descriptor); goto failed; @@ -1002,7 +1020,7 @@ static void iso_callback(struct fw_iso_context *context, u32 cycle, struct client *client = data; struct iso_interrupt_event *e; - e = kmalloc(sizeof(*e) + header_length, GFP_KERNEL); + e = kzalloc_flex(*e, interrupt.header, header_length); if (e == NULL) return; @@ -1327,30 +1345,27 @@ static void iso_resource_auto_work(struct work_struct *work) struct iso_resource_auto *r = from_work(r, work, work.work); struct client *client = r->client; unsigned long index = r->resource.handle; - int current_generation, resource_generation, channel, bandwidth, todo; - u64 reset_jiffies; + int channel, bandwidth; bool free; - scoped_guard(spinlock_irq, &client->lock) { - reset_jiffies = client->device->card->reset_jiffies; - current_generation = client->device->generation; - resource_generation = r->generation; - r->generation = current_generation; - todo = r->todo; - } + u64 reset_jiffies = client->device->card->reset_jiffies; + int current_generation = client->device->generation; + + int resource_generation = xchg(&r->generation, current_generation); // But no need to be atomic. + int todo = atomic_read(&r->todo); switch (todo) { case ISO_RES_AUTO_ALLOC: // Allow 1000ms grace period for other reallocations. if (time_is_after_jiffies64(reset_jiffies + secs_to_jiffies(1))) { schedule_iso_resource_auto(r, msecs_to_jiffies(333)); - goto out; + return; } break; case ISO_RES_AUTO_REALLOC: // We could be called twice within the same generation. if (resource_generation == current_generation) - goto out; + return; break; case ISO_RES_AUTO_DEALLOC: default: @@ -1364,23 +1379,23 @@ static void iso_resource_auto_work(struct work_struct *work) if (todo == ISO_RES_AUTO_DEALLOC) { free = true; - e = r->e_dealloc; - r->e_dealloc = NULL; + e = xchg(&r->e_dealloc, NULL); // But no need to be atomic. } else { free = false; // Is this generation outdated already? As long as this resource sticks in the // xarray, it will be scheduled again for a newer generation or at shutdown. if (channel == -EAGAIN) - goto out; + return; bool success = channel >= 0 || bandwidth > 0; if (!success) { // Allocation or reallocation failure? Pull this resource out of the // xarray and prepare for deletion, unless the client is shutting down. - scoped_guard(spinlock_irq, &client->lock) { + scoped_guard(mutex, &client->mutex) { if (!client->in_shutdown && xa_erase(&client->resource_xa, index)) { + // For the incrementation by add_client_resource(). client_put(client); free = true; } @@ -1389,24 +1404,20 @@ static void iso_resource_auto_work(struct work_struct *work) if (todo == ISO_RES_AUTO_REALLOC) { if (success) - goto out; + return; // Notify the userspace client of the failure through a deallocation event. - e = r->e_dealloc; - r->e_dealloc = NULL; + e = xchg(&r->e_dealloc, NULL); // But no need to be atomic. } else { - // Transit from allocation to reallocation, except if the client requested - // deallocation in the meantime. - scoped_guard(spinlock_irq, &client->lock) { - if (r->todo == ISO_RES_AUTO_ALLOC) - r->todo = ISO_RES_AUTO_REALLOC; - } + // Transit from allocation to reallocation. Use compare-and-swap atomic + // operation because the todo member can be set with ISO_RES_AUTO_DEALLOC + // by release_iso_resource_auto() in parallel. + atomic_cmpxchg_relaxed(&r->todo, ISO_RES_AUTO_ALLOC, ISO_RES_AUTO_REALLOC); if (channel >= 0) r->params.channels_mask = BIT_ULL(channel); - e = r->e_alloc; - r->e_alloc = NULL; + e = xchg(&r->e_alloc, NULL); // But no need to be atomic. } } @@ -1422,18 +1433,17 @@ static void iso_resource_auto_work(struct work_struct *work) kfree(r->e_alloc); kfree(r->e_dealloc); kfree(r); + + // For the incrementation by ioctl_allocate_iso_resource(). + client_put(client); } - out: - client_put(client); } static void release_iso_resource_auto(struct client *client, struct client_resource *resource) { struct iso_resource_auto *r = to_iso_resource_auto(resource); - guard(spinlock_irq)(&client->lock); - - r->todo = ISO_RES_AUTO_DEALLOC; + atomic_set(&r->todo, ISO_RES_AUTO_DEALLOC); schedule_iso_resource_auto(r, 0); } @@ -1454,7 +1464,7 @@ static int ioctl_allocate_iso_resource(struct client *client, union ioctl_arg *a INIT_DELAYED_WORK(&r->work, iso_resource_auto_work); r->client = client; - r->todo = ISO_RES_AUTO_ALLOC; + atomic_set(&r->todo, ISO_RES_AUTO_ALLOC); r->e_alloc = e1; r->e_dealloc = e2; @@ -1463,11 +1473,11 @@ static int ioctl_allocate_iso_resource(struct client *client, union ioctl_arg *a e2->iso_resource.closure = request->closure; e2->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED; - r->resource.release = release_iso_resource_auto; - err = add_client_resource(client, &r->resource, GFP_KERNEL); + err = add_client_resource(client, &r->resource, release_iso_resource_auto); if (err < 0) return err; request->handle = r->resource.handle; + client_get(client); retain_and_null_ptr(e1); retain_and_null_ptr(e2); @@ -1490,12 +1500,10 @@ static void iso_resource_once_work(struct work_struct *work) struct iso_resource_once *r = from_work(r, work, work); struct client *client = r->client; struct iso_resource_event *e = r->event; - int generation, channel, bandwidth; - - scoped_guard(spinlock_irq, &client->lock) - generation = client->device->generation; + int channel; - bandwidth = r->params.bandwidth; + int generation = client->device->generation; + int bandwidth = r->params.bandwidth; fw_iso_resource_manage(client->device->card, generation, r->params.channels_mask, &channel, &bandwidth, r->todo == ISO_RES_ONCE_ALLOC); @@ -1688,7 +1696,7 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg) if (!client->device->is_local) return -ENOSYS; - e = kzalloc(sizeof(*e) + sizeof(a->data), GFP_KERNEL); + e = kzalloc_flex(*e, phy_packet.with_tstamp.data, ARRAY_SIZE(a->data)); if (e == NULL) return -ENOMEM; @@ -1723,7 +1731,7 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg) trace_async_phy_outbound_initiate((uintptr_t)&e->p, card->index, e->p.generation, e->p.header[1], e->p.header[2]); - card->driver->send_request(card, &e->p); + smp_load_acquire(&card->driver)->send_request(card, &e->p); return 0; } @@ -1736,9 +1744,7 @@ static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg if (!client->device->is_local) return -ENOSYS; - // NOTE: This can be without irq when we can guarantee that __fw_send_request() for local - // destination never runs in any type of IRQ context. - scoped_guard(spinlock_irq, &phy_receiver_list_lock) + scoped_guard(mutex, &phy_receiver_list_mutex) list_move_tail(&client->phy_receiver_link, &phy_receiver_list); client->phy_receiver_closure = a->closure; @@ -1750,18 +1756,15 @@ void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p) { struct client *client; - // NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for local - // destination never runs in any type of IRQ context. - guard(spinlock_irqsave)(&phy_receiver_list_lock); + guard(mutex)(&phy_receiver_list_mutex); list_for_each_entry(client, &phy_receiver_list, phy_receiver_link) { - struct inbound_phy_packet_event *e; - if (client->device->card != card) continue; - e = kmalloc(sizeof(*e) + 8, GFP_ATOMIC); - if (e == NULL) + struct inbound_phy_packet_event *e = + kzalloc_flex(*e, phy_packet.with_tstamp.data, 2); + if (!e) break; if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { @@ -1911,11 +1914,11 @@ static bool has_outbound_transactions(struct client *client) struct client_resource *resource; unsigned long index; - guard(spinlock_irq)(&client->lock); - - xa_for_each(&client->resource_xa, index, resource) { - if (is_outbound_transaction_resource(resource)) - return true; + scoped_guard(mutex, &client->mutex) { + xa_for_each(&client->resource_xa, index, resource) { + if (is_outbound_transaction_resource(resource)) + return true; + } } return false; @@ -1928,9 +1931,7 @@ static int fw_device_op_release(struct inode *inode, struct file *file) struct client_resource *resource; unsigned long index; - // NOTE: This can be without irq when we can guarantee that __fw_send_request() for local - // destination never runs in any type of IRQ context. - scoped_guard(spinlock_irq, &phy_receiver_list_lock) + scoped_guard(mutex, &phy_receiver_list_mutex) list_del(&client->phy_receiver_link); scoped_guard(mutex, &client->device->client_list_mutex) @@ -1944,7 +1945,7 @@ static int fw_device_op_release(struct inode *inode, struct file *file) fw_iso_buffer_destroy(&client->buffer, client->device->card); // Freeze client->resource_xa and client->event_list. - scoped_guard(spinlock_irq, &client->lock) + scoped_guard(mutex, &client->mutex) client->in_shutdown = true; wait_event(client->tx_flush_wait, !has_outbound_transactions(client)); |
