xref: /linux/ipc/sem.c (revision 8116b54e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * linux/ipc/sem.c
4  * Copyright (C) 1992 Krishna Balasubramanian
5  * Copyright (C) 1995 Eric Schenk, Bruno Haible
6  *
7  * /proc/sysvipc/sem support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
8  *
9  * SMP-threaded, sysctl's added
10  * (c) 1999 Manfred Spraul <manfred@colorfullife.com>
11  * Enforced range limit on SEM_UNDO
12  * (c) 2001 Red Hat Inc
13  * Lockless wakeup
14  * (c) 2003 Manfred Spraul <manfred@colorfullife.com>
15  * (c) 2016 Davidlohr Bueso <dave@stgolabs.net>
16  * Further wakeup optimizations, documentation
17  * (c) 2010 Manfred Spraul <manfred@colorfullife.com>
18  *
19  * support for audit of ipc object properties and permission changes
20  * Dustin Kirkland <dustin.kirkland@us.ibm.com>
21  *
22  * namespaces support
23  * OpenVZ, SWsoft Inc.
24  * Pavel Emelianov <xemul@openvz.org>
25  *
26  * Implementation notes: (May 2010)
27  * This file implements System V semaphores.
28  *
29  * User space visible behavior:
30  * - FIFO ordering for semop() operations (just FIFO, not starvation
31  *   protection)
32  * - multiple semaphore operations that alter the same semaphore in
33  *   one semop() are handled.
34  * - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
35  *   SETALL calls.
36  * - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
37  * - undo adjustments at process exit are limited to 0..SEMVMX.
38  * - namespace are supported.
39  * - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtine by writing
40  *   to /proc/sys/kernel/sem.
41  * - statistics about the usage are reported in /proc/sysvipc/sem.
42  *
43  * Internals:
44  * - scalability:
45  *   - all global variables are read-mostly.
46  *   - semop() calls and semctl(RMID) are synchronized by RCU.
47  *   - most operations do write operations (actually: spin_lock calls) to
48  *     the per-semaphore array structure.
49  *   Thus: Perfect SMP scaling between independent semaphore arrays.
50  *         If multiple semaphores in one array are used, then cache line
51  *         trashing on the semaphore array spinlock will limit the scaling.
52  * - semncnt and semzcnt are calculated on demand in count_semcnt()
53  * - the task that performs a successful semop() scans the list of all
54  *   sleeping tasks and completes any pending operations that can be fulfilled.
55  *   Semaphores are actively given to waiting tasks (necessary for FIFO).
56  *   (see update_queue())
57  * - To improve the scalability, the actual wake-up calls are performed after
58  *   dropping all locks. (see wake_up_sem_queue_prepare())
59  * - All work is done by the waker, the woken up task does not have to do
60  *   anything - not even acquiring a lock or dropping a refcount.
61  * - A woken up task may not even touch the semaphore array anymore, it may
62  *   have been destroyed already by a semctl(RMID).
63  * - UNDO values are stored in an array (one per process and per
64  *   semaphore array, lazily allocated). For backwards compatibility, multiple
65  *   modes for the UNDO variables are supported (per process, per thread)
66  *   (see copy_semundo, CLONE_SYSVSEM)
67  * - There are two lists of the pending operations: a per-array list
68  *   and per-semaphore list (stored in the array). This allows to achieve FIFO
69  *   ordering without always scanning all pending operations.
70  *   The worst-case behavior is nevertheless O(N^2) for N wakeups.
71  */
72 
73 #include <linux/compat.h>
74 #include <linux/slab.h>
75 #include <linux/spinlock.h>
76 #include <linux/init.h>
77 #include <linux/proc_fs.h>
78 #include <linux/time.h>
79 #include <linux/security.h>
80 #include <linux/syscalls.h>
81 #include <linux/audit.h>
82 #include <linux/capability.h>
83 #include <linux/seq_file.h>
84 #include <linux/rwsem.h>
85 #include <linux/nsproxy.h>
86 #include <linux/ipc_namespace.h>
87 #include <linux/sched/wake_q.h>
88 #include <linux/nospec.h>
89 #include <linux/rhashtable.h>
90 
91 #include <linux/uaccess.h>
92 #include "util.h"
93 
94 /* One semaphore structure for each semaphore in the system. */
95 struct sem {
96 	int	semval;		/* current value */
97 	/*
98 	 * PID of the process that last modified the semaphore. For
99 	 * Linux, specifically these are:
100 	 *  - semop
101 	 *  - semctl, via SETVAL and SETALL.
102 	 *  - at task exit when performing undo adjustments (see exit_sem).
103 	 */
104 	struct pid *sempid;
105 	spinlock_t	lock;	/* spinlock for fine-grained semtimedop */
106 	struct list_head pending_alter; /* pending single-sop operations */
107 					/* that alter the semaphore */
108 	struct list_head pending_const; /* pending single-sop operations */
109 					/* that do not alter the semaphore*/
110 	time64_t	 sem_otime;	/* candidate for sem_otime */
111 } ____cacheline_aligned_in_smp;
112 
113 /* One sem_array data structure for each set of semaphores in the system. */
114 struct sem_array {
115 	struct kern_ipc_perm	sem_perm;	/* permissions .. see ipc.h */
116 	time64_t		sem_ctime;	/* create/last semctl() time */
117 	struct list_head	pending_alter;	/* pending operations */
118 						/* that alter the array */
119 	struct list_head	pending_const;	/* pending complex operations */
120 						/* that do not alter semvals */
121 	struct list_head	list_id;	/* undo requests on this array */
122 	int			sem_nsems;	/* no. of semaphores in array */
123 	int			complex_count;	/* pending complex operations */
124 	unsigned int		use_global_lock;/* >0: global lock required */
125 
126 	struct sem		sems[];
127 } __randomize_layout;
128 
129 /* One queue for each sleeping process in the system. */
130 struct sem_queue {
131 	struct list_head	list;	 /* queue of pending operations */
132 	struct task_struct	*sleeper; /* this process */
133 	struct sem_undo		*undo;	 /* undo structure */
134 	struct pid		*pid;	 /* process id of requesting process */
135 	int			status;	 /* completion status of operation */
136 	struct sembuf		*sops;	 /* array of pending operations */
137 	struct sembuf		*blocking; /* the operation that blocked */
138 	int			nsops;	 /* number of operations */
139 	bool			alter;	 /* does *sops alter the array? */
140 	bool                    dupsop;	 /* sops on more than one sem_num */
141 };
142 
143 /* Each task has a list of undo requests. They are executed automatically
144  * when the process exits.
145  */
146 struct sem_undo {
147 	struct list_head	list_proc;	/* per-process list: *
148 						 * all undos from one process
149 						 * rcu protected */
150 	struct rcu_head		rcu;		/* rcu struct for sem_undo */
151 	struct sem_undo_list	*ulp;		/* back ptr to sem_undo_list */
152 	struct list_head	list_id;	/* per semaphore array list:
153 						 * all undos for one array */
154 	int			semid;		/* semaphore set identifier */
155 	short			*semadj;	/* array of adjustments */
156 						/* one per semaphore */
157 };
158 
159 /* sem_undo_list controls shared access to the list of sem_undo structures
160  * that may be shared among all a CLONE_SYSVSEM task group.
161  */
162 struct sem_undo_list {
163 	refcount_t		refcnt;
164 	spinlock_t		lock;
165 	struct list_head	list_proc;
166 };
167 
168 
169 #define sem_ids(ns)	((ns)->ids[IPC_SEM_IDS])
170 
171 static int newary(struct ipc_namespace *, struct ipc_params *);
172 static void freeary(struct ipc_namespace *, struct kern_ipc_perm *);
173 #ifdef CONFIG_PROC_FS
174 static int sysvipc_sem_proc_show(struct seq_file *s, void *it);
175 #endif
176 
177 #define SEMMSL_FAST	256 /* 512 bytes on stack */
178 #define SEMOPM_FAST	64  /* ~ 372 bytes on stack */
179 
180 /*
181  * Switching from the mode suitable for simple ops
182  * to the mode for complex ops is costly. Therefore:
183  * use some hysteresis
184  */
185 #define USE_GLOBAL_LOCK_HYSTERESIS	10
186 
187 /*
188  * Locking:
189  * a) global sem_lock() for read/write
190  *	sem_undo.id_next,
191  *	sem_array.complex_count,
192  *	sem_array.pending{_alter,_const},
193  *	sem_array.sem_undo
194  *
195  * b) global or semaphore sem_lock() for read/write:
196  *	sem_array.sems[i].pending_{const,alter}:
197  *
198  * c) special:
199  *	sem_undo_list.list_proc:
200  *	* undo_list->lock for write
201  *	* rcu for read
202  *	use_global_lock:
203  *	* global sem_lock() for write
204  *	* either local or global sem_lock() for read.
205  *
206  * Memory ordering:
207  * Most ordering is enforced by using spin_lock() and spin_unlock().
208  *
209  * Exceptions:
210  * 1) use_global_lock: (SEM_BARRIER_1)
211  * Setting it from non-zero to 0 is a RELEASE, this is ensured by
212  * using smp_store_release(): Immediately after setting it to 0,
213  * a simple op can start.
214  * Testing if it is non-zero is an ACQUIRE, this is ensured by using
215  * smp_load_acquire().
216  * Setting it from 0 to non-zero must be ordered with regards to
217  * this smp_load_acquire(), this is guaranteed because the smp_load_acquire()
218  * is inside a spin_lock() and after a write from 0 to non-zero a
219  * spin_lock()+spin_unlock() is done.
220  *
221  * 2) queue.status: (SEM_BARRIER_2)
222  * Initialization is done while holding sem_lock(), so no further barrier is
223  * required.
224  * Setting it to a result code is a RELEASE, this is ensured by both a
225  * smp_store_release() (for case a) and while holding sem_lock()
226  * (for case b).
227  * The AQUIRE when reading the result code without holding sem_lock() is
228  * achieved by using READ_ONCE() + smp_acquire__after_ctrl_dep().
229  * (case a above).
230  * Reading the result code while holding sem_lock() needs no further barriers,
231  * the locks inside sem_lock() enforce ordering (case b above)
232  *
233  * 3) current->state:
234  * current->state is set to TASK_INTERRUPTIBLE while holding sem_lock().
235  * The wakeup is handled using the wake_q infrastructure. wake_q wakeups may
236  * happen immediately after calling wake_q_add. As wake_q_add_safe() is called
237  * when holding sem_lock(), no further barriers are required.
238  *
239  * See also ipc/mqueue.c for more details on the covered races.
240  */
241 
242 #define sc_semmsl	sem_ctls[0]
243 #define sc_semmns	sem_ctls[1]
244 #define sc_semopm	sem_ctls[2]
245 #define sc_semmni	sem_ctls[3]
246 
247 void sem_init_ns(struct ipc_namespace *ns)
248 {
249 	ns->sc_semmsl = SEMMSL;
250 	ns->sc_semmns = SEMMNS;
251 	ns->sc_semopm = SEMOPM;
252 	ns->sc_semmni = SEMMNI;
253 	ns->used_sems = 0;
254 	ipc_init_ids(&ns->ids[IPC_SEM_IDS]);
255 }
256 
257 #ifdef CONFIG_IPC_NS
258 void sem_exit_ns(struct ipc_namespace *ns)
259 {
260 	free_ipcs(ns, &sem_ids(ns), freeary);
261 	idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);
262 	rhashtable_destroy(&ns->ids[IPC_SEM_IDS].key_ht);
263 }
264 #endif
265 
266 void __init sem_init(void)
267 {
268 	sem_init_ns(&init_ipc_ns);
269 	ipc_init_proc_interface("sysvipc/sem",
270 				"       key      semid perms      nsems   uid   gid  cuid  cgid      otime      ctime\n",
271 				IPC_SEM_IDS, sysvipc_sem_proc_show);
272 }
273 
274 /**
275  * unmerge_queues - unmerge queues, if possible.
276  * @sma: semaphore array
277  *
278  * The function unmerges the wait queues if complex_count is 0.
279  * It must be called prior to dropping the global semaphore array lock.
280  */
281 static void unmerge_queues(struct sem_array *sma)
282 {
283 	struct sem_queue *q, *tq;
284 
285 	/* complex operations still around? */
286 	if (sma->complex_count)
287 		return;
288 	/*
289 	 * We will switch back to simple mode.
290 	 * Move all pending operation back into the per-semaphore
291 	 * queues.
292 	 */
293 	list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
294 		struct sem *curr;
295 		curr = &sma->sems[q->sops[0].sem_num];
296 
297 		list_add_tail(&q->list, &curr->pending_alter);
298 	}
299 	INIT_LIST_HEAD(&sma->pending_alter);
300 }
301 
302 /**
303  * merge_queues - merge single semop queues into global queue
304  * @sma: semaphore array
305  *
306  * This function merges all per-semaphore queues into the global queue.
307  * It is necessary to achieve FIFO ordering for the pending single-sop
308  * operations when a multi-semop operation must sleep.
309  * Only the alter operations must be moved, the const operations can stay.
310  */
311 static void merge_queues(struct sem_array *sma)
312 {
313 	int i;
314 	for (i = 0; i < sma->sem_nsems; i++) {
315 		struct sem *sem = &sma->sems[i];
316 
317 		list_splice_init(&sem->pending_alter, &sma->pending_alter);
318 	}
319 }
320 
321 static void sem_rcu_free(struct rcu_head *head)
322 {
323 	struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu);
324 	struct sem_array *sma = container_of(p, struct sem_array, sem_perm);
325 
326 	security_sem_free(&sma->sem_perm);
327 	kvfree(sma);
328 }
329 
330 /*
331  * Enter the mode suitable for non-simple operations:
332  * Caller must own sem_perm.lock.
333  */
334 static void complexmode_enter(struct sem_array *sma)
335 {
336 	int i;
337 	struct sem *sem;
338 
339 	if (sma->use_global_lock > 0)  {
340 		/*
341 		 * We are already in global lock mode.
342 		 * Nothing to do, just reset the
343 		 * counter until we return to simple mode.
344 		 */
345 		sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS;
346 		return;
347 	}
348 	sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS;
349 
350 	for (i = 0; i < sma->sem_nsems; i++) {
351 		sem = &sma->sems[i];
352 		spin_lock(&sem->lock);
353 		spin_unlock(&sem->lock);
354 	}
355 }
356 
357 /*
358  * Try to leave the mode that disallows simple operations:
359  * Caller must own sem_perm.lock.
360  */
361 static void complexmode_tryleave(struct sem_array *sma)
362 {
363 	if (sma->complex_count)  {
364 		/* Complex ops are sleeping.
365 		 * We must stay in complex mode
366 		 */
367 		return;
368 	}
369 	if (sma->use_global_lock == 1) {
370 
371 		/* See SEM_BARRIER_1 for purpose/pairing */
372 		smp_store_release(&sma->use_global_lock, 0);
373 	} else {
374 		sma->use_global_lock--;
375 	}
376 }
377 
378 #define SEM_GLOBAL_LOCK	(-1)
379 /*
380  * If the request contains only one semaphore operation, and there are
381  * no complex transactions pending, lock only the semaphore involved.
382  * Otherwise, lock the entire semaphore array, since we either have
383  * multiple semaphores in our own semops, or we need to look at
384  * semaphores from other pending complex operations.
385  */
386 static inline int sem_lock(struct sem_array *sma, struct sembuf *sops,
387 			      int nsops)
388 {
389 	struct sem *sem;
390 	int idx;
391 
392 	if (nsops != 1) {
393 		/* Complex operation - acquire a full lock */
394 		ipc_lock_object(&sma->sem_perm);
395 
396 		/* Prevent parallel simple ops */
397 		complexmode_enter(sma);
398 		return SEM_GLOBAL_LOCK;
399 	}
400 
401 	/*
402 	 * Only one semaphore affected - try to optimize locking.
403 	 * Optimized locking is possible if no complex operation
404 	 * is either enqueued or processed right now.
405 	 *
406 	 * Both facts are tracked by use_global_mode.
407 	 */
408 	idx = array_index_nospec(sops->sem_num, sma->sem_nsems);
409 	sem = &sma->sems[idx];
410 
411 	/*
412 	 * Initial check for use_global_lock. Just an optimization,
413 	 * no locking, no memory barrier.
414 	 */
415 	if (!sma->use_global_lock) {
416 		/*
417 		 * It appears that no complex operation is around.
418 		 * Acquire the per-semaphore lock.
419 		 */
420 		spin_lock(&sem->lock);
421 
422 		/* see SEM_BARRIER_1 for purpose/pairing */
423 		if (!smp_load_acquire(&sma->use_global_lock)) {
424 			/* fast path successful! */
425 			return sops->sem_num;
426 		}
427 		spin_unlock(&sem->lock);
428 	}
429 
430 	/* slow path: acquire the full lock */
431 	ipc_lock_object(&sma->sem_perm);
432 
433 	if (sma->use_global_lock == 0) {
434 		/*
435 		 * The use_global_lock mode ended while we waited for
436 		 * sma->sem_perm.lock. Thus we must switch to locking
437 		 * with sem->lock.
438 		 * Unlike in the fast path, there is no need to recheck
439 		 * sma->use_global_lock after we have acquired sem->lock:
440 		 * We own sma->sem_perm.lock, thus use_global_lock cannot
441 		 * change.
442 		 */
443 		spin_lock(&sem->lock);
444 
445 		ipc_unlock_object(&sma->sem_perm);
446 		return sops->sem_num;
447 	} else {
448 		/*
449 		 * Not a false alarm, thus continue to use the global lock
450 		 * mode. No need for complexmode_enter(), this was done by
451 		 * the caller that has set use_global_mode to non-zero.
452 		 */
453 		return SEM_GLOBAL_LOCK;
454 	}
455 }
456 
457 static inline void sem_unlock(struct sem_array *sma, int locknum)
458 {
459 	if (locknum == SEM_GLOBAL_LOCK) {
460 		unmerge_queues(sma);
461 		complexmode_tryleave(sma);
462 		ipc_unlock_object(&sma->sem_perm);
463 	} else {
464 		struct sem *sem = &sma->sems[locknum];
465 		spin_unlock(&sem->lock);
466 	}
467 }
468 
469 /*
470  * sem_lock_(check_) routines are called in the paths where the rwsem
471  * is not held.
472  *
473  * The caller holds the RCU read lock.
474  */
475 static inline struct sem_array *sem_obtain_object(struct ipc_namespace *ns, int id)
476 {
477 	struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&sem_ids(ns), id);
478 
479 	if (IS_ERR(ipcp))
480 		return ERR_CAST(ipcp);
481 
482 	return container_of(ipcp, struct sem_array, sem_perm);
483 }
484 
485 static inline struct sem_array *sem_obtain_object_check(struct ipc_namespace *ns,
486 							int id)
487 {
488 	struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&sem_ids(ns), id);
489 
490 	if (IS_ERR(ipcp))
491 		return ERR_CAST(ipcp);
492 
493 	return container_of(ipcp, struct sem_array, sem_perm);
494 }
495 
496 static inline void sem_lock_and_putref(struct sem_array *sma)
497 {
498 	sem_lock(sma, NULL, -1);
499 	ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
500 }
501 
502 static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s)
503 {
504 	ipc_rmid(&sem_ids(ns), &s->sem_perm);
505 }
506 
507 static struct sem_array *sem_alloc(size_t nsems)
508 {
509 	struct sem_array *sma;
510 
511 	if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0]))
512 		return NULL;
513 
514 	sma = kvzalloc(struct_size(sma, sems, nsems), GFP_KERNEL);
515 	if (unlikely(!sma))
516 		return NULL;
517 
518 	return sma;
519 }
520 
521 /**
522  * newary - Create a new semaphore set
523  * @ns: namespace
524  * @params: ptr to the structure that contains key, semflg and nsems
525  *
526  * Called with sem_ids.rwsem held (as a writer)
527  */
528 static int newary(struct ipc_namespace *ns, struct ipc_params *params)
529 {
530 	int retval;
531 	struct sem_array *sma;
532 	key_t key = params->key;
533 	int nsems = params->u.nsems;
534 	int semflg = params->flg;
535 	int i;
536 
537 	if (!nsems)
538 		return -EINVAL;
539 	if (ns->used_sems + nsems > ns->sc_semmns)
540 		return -ENOSPC;
541 
542 	sma = sem_alloc(nsems);
543 	if (!sma)
544 		return -ENOMEM;
545 
546 	sma->sem_perm.mode = (semflg & S_IRWXUGO);
547 	sma->sem_perm.key = key;
548 
549 	sma->sem_perm.security = NULL;
550 	retval = security_sem_alloc(&sma->sem_perm);
551 	if (retval) {
552 		kvfree(sma);
553 		return retval;
554 	}
555 
556 	for (i = 0; i < nsems; i++) {
557 		INIT_LIST_HEAD(&sma->sems[i].pending_alter);
558 		INIT_LIST_HEAD(&sma->sems[i].pending_const);
559 		spin_lock_init(&sma->sems[i].lock);
560 	}
561 
562 	sma->complex_count = 0;
563 	sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS;
564 	INIT_LIST_HEAD(&sma->pending_alter);
565 	INIT_LIST_HEAD(&sma->pending_const);
566 	INIT_LIST_HEAD(&sma->list_id);
567 	sma->sem_nsems = nsems;
568 	sma->sem_ctime = ktime_get_real_seconds();
569 
570 	/* ipc_addid() locks sma upon success. */
571 	retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni);
572 	if (retval < 0) {
573 		ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
574 		return retval;
575 	}
576 	ns->used_sems += nsems;
577 
578 	sem_unlock(sma, -1);
579 	rcu_read_unlock();
580 
581 	return sma->sem_perm.id;
582 }
583 
584 
585 /*
586  * Called with sem_ids.rwsem and ipcp locked.
587  */
588 static inline int sem_more_checks(struct kern_ipc_perm *ipcp,
589 				struct ipc_params *params)
590 {
591 	struct sem_array *sma;
592 
593 	sma = container_of(ipcp, struct sem_array, sem_perm);
594 	if (params->u.nsems > sma->sem_nsems)
595 		return -EINVAL;
596 
597 	return 0;
598 }
599 
600 long ksys_semget(key_t key, int nsems, int semflg)
601 {
602 	struct ipc_namespace *ns;
603 	static const struct ipc_ops sem_ops = {
604 		.getnew = newary,
605 		.associate = security_sem_associate,
606 		.more_checks = sem_more_checks,
607 	};
608 	struct ipc_params sem_params;
609 
610 	ns = current->nsproxy->ipc_ns;
611 
612 	if (nsems < 0 || nsems > ns->sc_semmsl)
613 		return -EINVAL;
614 
615 	sem_params.key = key;
616 	sem_params.flg = semflg;
617 	sem_params.u.nsems = nsems;
618 
619 	return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params);
620 }
621 
622 SYSCALL_DEFINE3(semget, key_t, key, int, nsems, int, semflg)
623 {
624 	return ksys_semget(key, nsems, semflg);
625 }
626 
627 /**
628  * perform_atomic_semop[_slow] - Attempt to perform semaphore
629  *                               operations on a given array.
630  * @sma: semaphore array
631  * @q: struct sem_queue that describes the operation
632  *
633  * Caller blocking are as follows, based the value
634  * indicated by the semaphore operation (sem_op):
635  *
636  *  (1) >0 never blocks.
637  *  (2)  0 (wait-for-zero operation): semval is non-zero.
638  *  (3) <0 attempting to decrement semval to a value smaller than zero.
639  *
640  * Returns 0 if the operation was possible.
641  * Returns 1 if the operation is impossible, the caller must sleep.
642  * Returns <0 for error codes.
643  */
644 static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q)
645 {
646 	int result, sem_op, nsops;
647 	struct pid *pid;
648 	struct sembuf *sop;
649 	struct sem *curr;
650 	struct sembuf *sops;
651 	struct sem_undo *un;
652 
653 	sops = q->sops;
654 	nsops = q->nsops;
655 	un = q->undo;
656 
657 	for (sop = sops; sop < sops + nsops; sop++) {
658 		int idx = array_index_nospec(sop->sem_num, sma->sem_nsems);
659 		curr = &sma->sems[idx];
660 		sem_op = sop->sem_op;
661 		result = curr->semval;
662 
663 		if (!sem_op && result)
664 			goto would_block;
665 
666 		result += sem_op;
667 		if (result < 0)
668 			goto would_block;
669 		if (result > SEMVMX)
670 			goto out_of_range;
671 
672 		if (sop->sem_flg & SEM_UNDO) {
673 			int undo = un->semadj[sop->sem_num] - sem_op;
674 			/* Exceeding the undo range is an error. */
675 			if (undo < (-SEMAEM - 1) || undo > SEMAEM)
676 				goto out_of_range;
677 			un->semadj[sop->sem_num] = undo;
678 		}
679 
680 		curr->semval = result;
681 	}
682 
683 	sop--;
684 	pid = q->pid;
685 	while (sop >= sops) {
686 		ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid);
687 		sop--;
688 	}
689 
690 	return 0;
691 
692 out_of_range:
693 	result = -ERANGE;
694 	goto undo;
695 
696 would_block:
697 	q->blocking = sop;
698 
699 	if (sop->sem_flg & IPC_NOWAIT)
700 		result = -EAGAIN;
701 	else
702 		result = 1;
703 
704 undo:
705 	sop--;
706 	while (sop >= sops) {
707 		sem_op = sop->sem_op;
708 		sma->sems[sop->sem_num].semval -= sem_op;
709 		if (sop->sem_flg & SEM_UNDO)
710 			un->semadj[sop->sem_num] += sem_op;
711 		sop--;
712 	}
713 
714 	return result;
715 }
716 
717 static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q)
718 {
719 	int result, sem_op, nsops;
720 	struct sembuf *sop;
721 	struct sem *curr;
722 	struct sembuf *sops;
723 	struct sem_undo *un;
724 
725 	sops = q->sops;
726 	nsops = q->nsops;
727 	un = q->undo;
728 
729 	if (unlikely(q->dupsop))
730 		return perform_atomic_semop_slow(sma, q);
731 
732 	/*
733 	 * We scan the semaphore set twice, first to ensure that the entire
734 	 * operation can succeed, therefore avoiding any pointless writes
735 	 * to shared memory and having to undo such changes in order to block
736 	 * until the operations can go through.
737 	 */
738 	for (sop = sops; sop < sops + nsops; sop++) {
739 		int idx = array_index_nospec(sop->sem_num, sma->sem_nsems);
740 
741 		curr = &sma->sems[idx];
742 		sem_op = sop->sem_op;
743 		result = curr->semval;
744 
745 		if (!sem_op && result)
746 			goto would_block; /* wait-for-zero */
747 
748 		result += sem_op;
749 		if (result < 0)
750 			goto would_block;
751 
752 		if (result > SEMVMX)
753 			return -ERANGE;
754 
755 		if (sop->sem_flg & SEM_UNDO) {
756 			int undo = un->semadj[sop->sem_num] - sem_op;
757 
758 			/* Exceeding the undo range is an error. */
759 			if (undo < (-SEMAEM - 1) || undo > SEMAEM)
760 				return -ERANGE;
761 		}
762 	}
763 
764 	for (sop = sops; sop < sops + nsops; sop++) {
765 		curr = &sma->sems[sop->sem_num];
766 		sem_op = sop->sem_op;
767 		result = curr->semval;
768 
769 		if (sop->sem_flg & SEM_UNDO) {
770 			int undo = un->semadj[sop->sem_num] - sem_op;
771 
772 			un->semadj[sop->sem_num] = undo;
773 		}
774 		curr->semval += sem_op;
775 		ipc_update_pid(&curr->sempid, q->pid);
776 	}
777 
778 	return 0;
779 
780 would_block:
781 	q->blocking = sop;
782 	return sop->sem_flg & IPC_NOWAIT ? -EAGAIN : 1;
783 }
784 
785 static inline void wake_up_sem_queue_prepare(struct sem_queue *q, int error,
786 					     struct wake_q_head *wake_q)
787 {
788 	get_task_struct(q->sleeper);
789 
790 	/* see SEM_BARRIER_2 for purpuse/pairing */
791 	smp_store_release(&q->status, error);
792 
793 	wake_q_add_safe(wake_q, q->sleeper);
794 }
795 
796 static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
797 {
798 	list_del(&q->list);
799 	if (q->nsops > 1)
800 		sma->complex_count--;
801 }
802 
803 /** check_restart(sma, q)
804  * @sma: semaphore array
805  * @q: the operation that just completed
806  *
807  * update_queue is O(N^2) when it restarts scanning the whole queue of
808  * waiting operations. Therefore this function checks if the restart is
809  * really necessary. It is called after a previously waiting operation
810  * modified the array.
811  * Note that wait-for-zero operations are handled without restart.
812  */
813 static inline int check_restart(struct sem_array *sma, struct sem_queue *q)
814 {
815 	/* pending complex alter operations are too difficult to analyse */
816 	if (!list_empty(&sma->pending_alter))
817 		return 1;
818 
819 	/* we were a sleeping complex operation. Too difficult */
820 	if (q->nsops > 1)
821 		return 1;
822 
823 	/* It is impossible that someone waits for the new value:
824 	 * - complex operations always restart.
825 	 * - wait-for-zero are handled seperately.
826 	 * - q is a previously sleeping simple operation that
827 	 *   altered the array. It must be a decrement, because
828 	 *   simple increments never sleep.
829 	 * - If there are older (higher priority) decrements
830 	 *   in the queue, then they have observed the original
831 	 *   semval value and couldn't proceed. The operation
832 	 *   decremented to value - thus they won't proceed either.
833 	 */
834 	return 0;
835 }
836 
837 /**
838  * wake_const_ops - wake up non-alter tasks
839  * @sma: semaphore array.
840  * @semnum: semaphore that was modified.
841  * @wake_q: lockless wake-queue head.
842  *
843  * wake_const_ops must be called after a semaphore in a semaphore array
844  * was set to 0. If complex const operations are pending, wake_const_ops must
845  * be called with semnum = -1, as well as with the number of each modified
846  * semaphore.
847  * The tasks that must be woken up are added to @wake_q. The return code
848  * is stored in q->pid.
849  * The function returns 1 if at least one operation was completed successfully.
850  */
851 static int wake_const_ops(struct sem_array *sma, int semnum,
852 			  struct wake_q_head *wake_q)
853 {
854 	struct sem_queue *q, *tmp;
855 	struct list_head *pending_list;
856 	int semop_completed = 0;
857 
858 	if (semnum == -1)
859 		pending_list = &sma->pending_const;
860 	else
861 		pending_list = &sma->sems[semnum].pending_const;
862 
863 	list_for_each_entry_safe(q, tmp, pending_list, list) {
864 		int error = perform_atomic_semop(sma, q);
865 
866 		if (error > 0)
867 			continue;
868 		/* operation completed, remove from queue & wakeup */
869 		unlink_queue(sma, q);
870 
871 		wake_up_sem_queue_prepare(q, error, wake_q);
872 		if (error == 0)
873 			semop_completed = 1;
874 	}
875 
876 	return semop_completed;
877 }
878 
879 /**
880  * do_smart_wakeup_zero - wakeup all wait for zero tasks
881  * @sma: semaphore array
882  * @sops: operations that were performed
883  * @nsops: number of operations
884  * @wake_q: lockless wake-queue head
885  *
886  * Checks all required queue for wait-for-zero operations, based
887  * on the actual changes that were performed on the semaphore array.
888  * The function returns 1 if at least one operation was completed successfully.
889  */
890 static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
891 				int nsops, struct wake_q_head *wake_q)
892 {
893 	int i;
894 	int semop_completed = 0;
895 	int got_zero = 0;
896 
897 	/* first: the per-semaphore queues, if known */
898 	if (sops) {
899 		for (i = 0; i < nsops; i++) {
900 			int num = sops[i].sem_num;
901 
902 			if (sma->sems[num].semval == 0) {
903 				got_zero = 1;
904 				semop_completed |= wake_const_ops(sma, num, wake_q);
905 			}
906 		}
907 	} else {
908 		/*
909 		 * No sops means modified semaphores not known.
910 		 * Assume all were changed.
911 		 */
912 		for (i = 0; i < sma->sem_nsems; i++) {
913 			if (sma->sems[i].semval == 0) {
914 				got_zero = 1;
915 				semop_completed |= wake_const_ops(sma, i, wake_q);
916 			}
917 		}
918 	}
919 	/*
920 	 * If one of the modified semaphores got 0,
921 	 * then check the global queue, too.
922 	 */
923 	if (got_zero)
924 		semop_completed |= wake_const_ops(sma, -1, wake_q);
925 
926 	return semop_completed;
927 }
928 
929 
930 /**
931  * update_queue - look for tasks that can be completed.
932  * @sma: semaphore array.
933  * @semnum: semaphore that was modified.
934  * @wake_q: lockless wake-queue head.
935  *
936  * update_queue must be called after a semaphore in a semaphore array
937  * was modified. If multiple semaphores were modified, update_queue must
938  * be called with semnum = -1, as well as with the number of each modified
939  * semaphore.
940  * The tasks that must be woken up are added to @wake_q. The return code
941  * is stored in q->pid.
942  * The function internally checks if const operations can now succeed.
943  *
944  * The function return 1 if at least one semop was completed successfully.
945  */
946 static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q)
947 {
948 	struct sem_queue *q, *tmp;
949 	struct list_head *pending_list;
950 	int semop_completed = 0;
951 
952 	if (semnum == -1)
953 		pending_list = &sma->pending_alter;
954 	else
955 		pending_list = &sma->sems[semnum].pending_alter;
956 
957 again:
958 	list_for_each_entry_safe(q, tmp, pending_list, list) {
959 		int error, restart;
960 
961 		/* If we are scanning the single sop, per-semaphore list of
962 		 * one semaphore and that semaphore is 0, then it is not
963 		 * necessary to scan further: simple increments
964 		 * that affect only one entry succeed immediately and cannot
965 		 * be in the  per semaphore pending queue, and decrements
966 		 * cannot be successful if the value is already 0.
967 		 */
968 		if (semnum != -1 && sma->sems[semnum].semval == 0)
969 			break;
970 
971 		error = perform_atomic_semop(sma, q);
972 
973 		/* Does q->sleeper still need to sleep? */
974 		if (error > 0)
975 			continue;
976 
977 		unlink_queue(sma, q);
978 
979 		if (error) {
980 			restart = 0;
981 		} else {
982 			semop_completed = 1;
983 			do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q);
984 			restart = check_restart(sma, q);
985 		}
986 
987 		wake_up_sem_queue_prepare(q, error, wake_q);
988 		if (restart)
989 			goto again;
990 	}
991 	return semop_completed;
992 }
993 
994 /**
995  * set_semotime - set sem_otime
996  * @sma: semaphore array
997  * @sops: operations that modified the array, may be NULL
998  *
999  * sem_otime is replicated to avoid cache line trashing.
1000  * This function sets one instance to the current time.
1001  */
1002 static void set_semotime(struct sem_array *sma, struct sembuf *sops)
1003 {
1004 	if (sops == NULL) {
1005 		sma->sems[0].sem_otime = ktime_get_real_seconds();
1006 	} else {
1007 		sma->sems[sops[0].sem_num].sem_otime =
1008 						ktime_get_real_seconds();
1009 	}
1010 }
1011 
1012 /**
1013  * do_smart_update - optimized update_queue
1014  * @sma: semaphore array
1015  * @sops: operations that were performed
1016  * @nsops: number of operations
1017  * @otime: force setting otime
1018  * @wake_q: lockless wake-queue head
1019  *
1020  * do_smart_update() does the required calls to update_queue and wakeup_zero,
1021  * based on the actual changes that were performed on the semaphore array.
1022  * Note that the function does not do the actual wake-up: the caller is
1023  * responsible for calling wake_up_q().
1024  * It is safe to perform this call after dropping all locks.
1025  */
1026 static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops,
1027 			    int otime, struct wake_q_head *wake_q)
1028 {
1029 	int i;
1030 
1031 	otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q);
1032 
1033 	if (!list_empty(&sma->pending_alter)) {
1034 		/* semaphore array uses the global queue - just process it. */
1035 		otime |= update_queue(sma, -1, wake_q);
1036 	} else {
1037 		if (!sops) {
1038 			/*
1039 			 * No sops, thus the modified semaphores are not
1040 			 * known. Check all.
1041 			 */
1042 			for (i = 0; i < sma->sem_nsems; i++)
1043 				otime |= update_queue(sma, i, wake_q);
1044 		} else {
1045 			/*
1046 			 * Check the semaphores that were increased:
1047 			 * - No complex ops, thus all sleeping ops are
1048 			 *   decrease.
1049 			 * - if we decreased the value, then any sleeping
1050 			 *   semaphore ops wont be able to run: If the
1051 			 *   previous value was too small, then the new
1052 			 *   value will be too small, too.
1053 			 */
1054 			for (i = 0; i < nsops; i++) {
1055 				if (sops[i].sem_op > 0) {
1056 					otime |= update_queue(sma,
1057 							      sops[i].sem_num, wake_q);
1058 				}
1059 			}
1060 		}
1061 	}
1062 	if (otime)
1063 		set_semotime(sma, sops);
1064 }
1065 
1066 /*
1067  * check_qop: Test if a queued operation sleeps on the semaphore semnum
1068  */
1069 static int check_qop(struct sem_array *sma, int semnum, struct sem_queue *q,
1070 			bool count_zero)
1071 {
1072 	struct sembuf *sop = q->blocking;
1073 
1074 	/*
1075 	 * Linux always (since 0.99.10) reported a task as sleeping on all
1076 	 * semaphores. This violates SUS, therefore it was changed to the
1077 	 * standard compliant behavior.
1078 	 * Give the administrators a chance to notice that an application
1079 	 * might misbehave because it relies on the Linux behavior.
1080 	 */
1081 	pr_info_once("semctl(GETNCNT/GETZCNT) is since 3.16 Single Unix Specification compliant.\n"
1082 			"The task %s (%d) triggered the difference, watch for misbehavior.\n",
1083 			current->comm, task_pid_nr(current));
1084 
1085 	if (sop->sem_num != semnum)
1086 		return 0;
1087 
1088 	if (count_zero && sop->sem_op == 0)
1089 		return 1;
1090 	if (!count_zero && sop->sem_op < 0)
1091 		return 1;
1092 
1093 	return 0;
1094 }
1095 
1096 /* The following counts are associated to each semaphore:
1097  *   semncnt        number of tasks waiting on semval being nonzero
1098  *   semzcnt        number of tasks waiting on semval being zero
1099  *
1100  * Per definition, a task waits only on the semaphore of the first semop
1101  * that cannot proceed, even if additional operation would block, too.
1102  */
1103 static int count_semcnt(struct sem_array *sma, ushort semnum,
1104 			bool count_zero)
1105 {
1106 	struct list_head *l;
1107 	struct sem_queue *q;
1108 	int semcnt;
1109 
1110 	semcnt = 0;
1111 	/* First: check the simple operations. They are easy to evaluate */
1112 	if (count_zero)
1113 		l = &sma->sems[semnum].pending_const;
1114 	else
1115 		l = &sma->sems[semnum].pending_alter;
1116 
1117 	list_for_each_entry(q, l, list) {
1118 		/* all task on a per-semaphore list sleep on exactly
1119 		 * that semaphore
1120 		 */
1121 		semcnt++;
1122 	}
1123 
1124 	/* Then: check the complex operations. */
1125 	list_for_each_entry(q, &sma->pending_alter, list) {
1126 		semcnt += check_qop(sma, semnum, q, count_zero);
1127 	}
1128 	if (count_zero) {
1129 		list_for_each_entry(q, &sma->pending_const, list) {
1130 			semcnt += check_qop(sma, semnum, q, count_zero);
1131 		}
1132 	}
1133 	return semcnt;
1134 }
1135 
1136 /* Free a semaphore set. freeary() is called with sem_ids.rwsem locked
1137  * as a writer and the spinlock for this semaphore set hold. sem_ids.rwsem
1138  * remains locked on exit.
1139  */
1140 static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
1141 {
1142 	struct sem_undo *un, *tu;
1143 	struct sem_queue *q, *tq;
1144 	struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
1145 	int i;
1146 	DEFINE_WAKE_Q(wake_q);
1147 
1148 	/* Free the existing undo structures for this semaphore set.  */
1149 	ipc_assert_locked_object(&sma->sem_perm);
1150 	list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {
1151 		list_del(&un->list_id);
1152 		spin_lock(&un->ulp->lock);
1153 		un->semid = -1;
1154 		list_del_rcu(&un->list_proc);
1155 		spin_unlock(&un->ulp->lock);
1156 		kfree_rcu(un, rcu);
1157 	}
1158 
1159 	/* Wake up all pending processes and let them fail with EIDRM. */
1160 	list_for_each_entry_safe(q, tq, &sma->pending_const, list) {
1161 		unlink_queue(sma, q);
1162 		wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
1163 	}
1164 
1165 	list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
1166 		unlink_queue(sma, q);
1167 		wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
1168 	}
1169 	for (i = 0; i < sma->sem_nsems; i++) {
1170 		struct sem *sem = &sma->sems[i];
1171 		list_for_each_entry_safe(q, tq, &sem->pending_const, list) {
1172 			unlink_queue(sma, q);
1173 			wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
1174 		}
1175 		list_for_each_entry_safe(q, tq, &sem->pending_alter, list) {
1176 			unlink_queue(sma, q);
1177 			wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
1178 		}
1179 		ipc_update_pid(&sem->sempid, NULL);
1180 	}
1181 
1182 	/* Remove the semaphore set from the IDR */
1183 	sem_rmid(ns, sma);
1184 	sem_unlock(sma, -1);
1185 	rcu_read_unlock();
1186 
1187 	wake_up_q(&wake_q);
1188 	ns->used_sems -= sma->sem_nsems;
1189 	ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
1190 }
1191 
1192 static unsigned long copy_semid_to_user(void __user *buf, struct semid64_ds *in, int version)
1193 {
1194 	switch (version) {
1195 	case IPC_64:
1196 		return copy_to_user(buf, in, sizeof(*in));
1197 	case IPC_OLD:
1198 	    {
1199 		struct semid_ds out;
1200 
1201 		memset(&out, 0, sizeof(out));
1202 
1203 		ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm);
1204 
1205 		out.sem_otime	= in->sem_otime;
1206 		out.sem_ctime	= in->sem_ctime;
1207 		out.sem_nsems	= in->sem_nsems;
1208 
1209 		return copy_to_user(buf, &out, sizeof(out));
1210 	    }
1211 	default:
1212 		return -EINVAL;
1213 	}
1214 }
1215 
1216 static time64_t get_semotime(struct sem_array *sma)
1217 {
1218 	int i;
1219 	time64_t res;
1220 
1221 	res = sma->sems[0].sem_otime;
1222 	for (i = 1; i < sma->sem_nsems; i++) {
1223 		time64_t to = sma->sems[i].sem_otime;
1224 
1225 		if (to > res)
1226 			res = to;
1227 	}
1228 	return res;
1229 }
1230 
1231 static int semctl_stat(struct ipc_namespace *ns, int semid,
1232 			 int cmd, struct semid64_ds *semid64)
1233 {
1234 	struct sem_array *sma;
1235 	time64_t semotime;
1236 	int err;
1237 
1238 	memset(semid64, 0, sizeof(*semid64));
1239 
1240 	rcu_read_lock();
1241 	if (cmd == SEM_STAT || cmd == SEM_STAT_ANY) {
1242 		sma = sem_obtain_object(ns, semid);
1243 		if (IS_ERR(sma)) {
1244 			err = PTR_ERR(sma);
1245 			goto out_unlock;
1246 		}
1247 	} else { /* IPC_STAT */
1248 		sma = sem_obtain_object_check(ns, semid);
1249 		if (IS_ERR(sma)) {
1250 			err = PTR_ERR(sma);
1251 			goto out_unlock;
1252 		}
1253 	}
1254 
1255 	/* see comment for SHM_STAT_ANY */
1256 	if (cmd == SEM_STAT_ANY)
1257 		audit_ipc_obj(&sma->sem_perm);
1258 	else {
1259 		err = -EACCES;
1260 		if (ipcperms(ns, &sma->sem_perm, S_IRUGO))
1261 			goto out_unlock;
1262 	}
1263 
1264 	err = security_sem_semctl(&sma->sem_perm, cmd);
1265 	if (err)
1266 		goto out_unlock;
1267 
1268 	ipc_lock_object(&sma->sem_perm);
1269 
1270 	if (!ipc_valid_object(&sma->sem_perm)) {
1271 		ipc_unlock_object(&sma->sem_perm);
1272 		err = -EIDRM;
1273 		goto out_unlock;
1274 	}
1275 
1276 	kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm);
1277 	semotime = get_semotime(sma);
1278 	semid64->sem_otime = semotime;
1279 	semid64->sem_ctime = sma->sem_ctime;
1280 #ifndef CONFIG_64BIT
1281 	semid64->sem_otime_high = semotime >> 32;
1282 	semid64->sem_ctime_high = sma->sem_ctime >> 32;
1283 #endif
1284 	semid64->sem_nsems = sma->sem_nsems;
1285 
1286 	if (cmd == IPC_STAT) {
1287 		/*
1288 		 * As defined in SUS:
1289 		 * Return 0 on success
1290 		 */
1291 		err = 0;
1292 	} else {
1293 		/*
1294 		 * SEM_STAT and SEM_STAT_ANY (both Linux specific)
1295 		 * Return the full id, including the sequence number
1296 		 */
1297 		err = sma->sem_perm.id;
1298 	}
1299 	ipc_unlock_object(&sma->sem_perm);
1300 out_unlock:
1301 	rcu_read_unlock();
1302 	return err;
1303 }
1304 
1305 static int semctl_info(struct ipc_namespace *ns, int semid,
1306 			 int cmd, void __user *p)
1307 {
1308 	struct seminfo seminfo;
1309 	int max_idx;
1310 	int err;
1311 
1312 	err = security_sem_semctl(NULL, cmd);
1313 	if (err)
1314 		return err;
1315 
1316 	memset(&seminfo, 0, sizeof(seminfo));
1317 	seminfo.semmni = ns->sc_semmni;
1318 	seminfo.semmns = ns->sc_semmns;
1319 	seminfo.semmsl = ns->sc_semmsl;
1320 	seminfo.semopm = ns->sc_semopm;
1321 	seminfo.semvmx = SEMVMX;
1322 	seminfo.semmnu = SEMMNU;
1323 	seminfo.semmap = SEMMAP;
1324 	seminfo.semume = SEMUME;
1325 	down_read(&sem_ids(ns).rwsem);
1326 	if (cmd == SEM_INFO) {
1327 		seminfo.semusz = sem_ids(ns).in_use;
1328 		seminfo.semaem = ns->used_sems;
1329 	} else {
1330 		seminfo.semusz = SEMUSZ;
1331 		seminfo.semaem = SEMAEM;
1332 	}
1333 	max_idx = ipc_get_maxidx(&sem_ids(ns));
1334 	up_read(&sem_ids(ns).rwsem);
1335 	if (copy_to_user(p, &seminfo, sizeof(struct seminfo)))
1336 		return -EFAULT;
1337 	return (max_idx < 0) ? 0 : max_idx;
1338 }
1339 
1340 static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum,
1341 		int val)
1342 {
1343 	struct sem_undo *un;
1344 	struct sem_array *sma;
1345 	struct sem *curr;
1346 	int err;
1347 	DEFINE_WAKE_Q(wake_q);
1348 
1349 	if (val > SEMVMX || val < 0)
1350 		return -ERANGE;
1351 
1352 	rcu_read_lock();
1353 	sma = sem_obtain_object_check(ns, semid);
1354 	if (IS_ERR(sma)) {
1355 		rcu_read_unlock();
1356 		return PTR_ERR(sma);
1357 	}
1358 
1359 	if (semnum < 0 || semnum >= sma->sem_nsems) {
1360 		rcu_read_unlock();
1361 		return -EINVAL;
1362 	}
1363 
1364 
1365 	if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) {
1366 		rcu_read_unlock();
1367 		return -EACCES;
1368 	}
1369 
1370 	err = security_sem_semctl(&sma->sem_perm, SETVAL);
1371 	if (err) {
1372 		rcu_read_unlock();
1373 		return -EACCES;
1374 	}
1375 
1376 	sem_lock(sma, NULL, -1);
1377 
1378 	if (!ipc_valid_object(&sma->sem_perm)) {
1379 		sem_unlock(sma, -1);
1380 		rcu_read_unlock();
1381 		return -EIDRM;
1382 	}
1383 
1384 	semnum = array_index_nospec(semnum, sma->sem_nsems);
1385 	curr = &sma->sems[semnum];
1386 
1387 	ipc_assert_locked_object(&sma->sem_perm);
1388 	list_for_each_entry(un, &sma->list_id, list_id)
1389 		un->semadj[semnum] = 0;
1390 
1391 	curr->semval = val;
1392 	ipc_update_pid(&curr->sempid, task_tgid(current));
1393 	sma->sem_ctime = ktime_get_real_seconds();
1394 	/* maybe some queued-up processes were waiting for this */
1395 	do_smart_update(sma, NULL, 0, 0, &wake_q);
1396 	sem_unlock(sma, -1);
1397 	rcu_read_unlock();
1398 	wake_up_q(&wake_q);
1399 	return 0;
1400 }
1401 
1402 static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
1403 		int cmd, void __user *p)
1404 {
1405 	struct sem_array *sma;
1406 	struct sem *curr;
1407 	int err, nsems;
1408 	ushort fast_sem_io[SEMMSL_FAST];
1409 	ushort *sem_io = fast_sem_io;
1410 	DEFINE_WAKE_Q(wake_q);
1411 
1412 	rcu_read_lock();
1413 	sma = sem_obtain_object_check(ns, semid);
1414 	if (IS_ERR(sma)) {
1415 		rcu_read_unlock();
1416 		return PTR_ERR(sma);
1417 	}
1418 
1419 	nsems = sma->sem_nsems;
1420 
1421 	err = -EACCES;
1422 	if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO))
1423 		goto out_rcu_wakeup;
1424 
1425 	err = security_sem_semctl(&sma->sem_perm, cmd);
1426 	if (err)
1427 		goto out_rcu_wakeup;
1428 
1429 	err = -EACCES;
1430 	switch (cmd) {
1431 	case GETALL:
1432 	{
1433 		ushort __user *array = p;
1434 		int i;
1435 
1436 		sem_lock(sma, NULL, -1);
1437 		if (!ipc_valid_object(&sma->sem_perm)) {
1438 			err = -EIDRM;
1439 			goto out_unlock;
1440 		}
1441 		if (nsems > SEMMSL_FAST) {
1442 			if (!ipc_rcu_getref(&sma->sem_perm)) {
1443 				err = -EIDRM;
1444 				goto out_unlock;
1445 			}
1446 			sem_unlock(sma, -1);
1447 			rcu_read_unlock();
1448 			sem_io = kvmalloc_array(nsems, sizeof(ushort),
1449 						GFP_KERNEL);
1450 			if (sem_io == NULL) {
1451 				ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
1452 				return -ENOMEM;
1453 			}
1454 
1455 			rcu_read_lock();
1456 			sem_lock_and_putref(sma);
1457 			if (!ipc_valid_object(&sma->sem_perm)) {
1458 				err = -EIDRM;
1459 				goto out_unlock;
1460 			}
1461 		}
1462 		for (i = 0; i < sma->sem_nsems; i++)
1463 			sem_io[i] = sma->sems[i].semval;
1464 		sem_unlock(sma, -1);
1465 		rcu_read_unlock();
1466 		err = 0;
1467 		if (copy_to_user(array, sem_io, nsems*sizeof(ushort)))
1468 			err = -EFAULT;
1469 		goto out_free;
1470 	}
1471 	case SETALL:
1472 	{
1473 		int i;
1474 		struct sem_undo *un;
1475 
1476 		if (!ipc_rcu_getref(&sma->sem_perm)) {
1477 			err = -EIDRM;
1478 			goto out_rcu_wakeup;
1479 		}
1480 		rcu_read_unlock();
1481 
1482 		if (nsems > SEMMSL_FAST) {
1483 			sem_io = kvmalloc_array(nsems, sizeof(ushort),
1484 						GFP_KERNEL);
1485 			if (sem_io == NULL) {
1486 				ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
1487 				return -ENOMEM;
1488 			}
1489 		}
1490 
1491 		if (copy_from_user(sem_io, p, nsems*sizeof(ushort))) {
1492 			ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
1493 			err = -EFAULT;
1494 			goto out_free;
1495 		}
1496 
1497 		for (i = 0; i < nsems; i++) {
1498 			if (sem_io[i] > SEMVMX) {
1499 				ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
1500 				err = -ERANGE;
1501 				goto out_free;
1502 			}
1503 		}
1504 		rcu_read_lock();
1505 		sem_lock_and_putref(sma);
1506 		if (!ipc_valid_object(&sma->sem_perm)) {
1507 			err = -EIDRM;
1508 			goto out_unlock;
1509 		}
1510 
1511 		for (i = 0; i < nsems; i++) {
1512 			sma->sems[i].semval = sem_io[i];
1513 			ipc_update_pid(&sma->sems[i].sempid, task_tgid(current));
1514 		}
1515 
1516 		ipc_assert_locked_object(&sma->sem_perm);
1517 		list_for_each_entry(un, &sma->list_id, list_id) {
1518 			for (i = 0; i < nsems; i++)
1519 				un->semadj[i] = 0;
1520 		}
1521 		sma->sem_ctime = ktime_get_real_seconds();
1522 		/* maybe some queued-up processes were waiting for this */
1523 		do_smart_update(sma, NULL, 0, 0, &wake_q);
1524 		err = 0;
1525 		goto out_unlock;
1526 	}
1527 	/* GETVAL, GETPID, GETNCTN, GETZCNT: fall-through */
1528 	}
1529 	err = -EINVAL;
1530 	if (semnum < 0 || semnum >= nsems)
1531 		goto out_rcu_wakeup;
1532 
1533 	sem_lock(sma, NULL, -1);
1534 	if (!ipc_valid_object(&sma->sem_perm)) {
1535 		err = -EIDRM;
1536 		goto out_unlock;
1537 	}
1538 
1539 	semnum = array_index_nospec(semnum, nsems);
1540 	curr = &sma->sems[semnum];
1541 
1542 	switch (cmd) {
1543 	case GETVAL:
1544 		err = curr->semval;
1545 		goto out_unlock;
1546 	case GETPID:
1547 		err = pid_vnr(curr->sempid);
1548 		goto out_unlock;
1549 	case GETNCNT:
1550 		err = count_semcnt(sma, semnum, 0);
1551 		goto out_unlock;
1552 	case GETZCNT:
1553 		err = count_semcnt(sma, semnum, 1);
1554 		goto out_unlock;
1555 	}
1556 
1557 out_unlock:
1558 	sem_unlock(sma, -1);
1559 out_rcu_wakeup:
1560 	rcu_read_unlock();
1561 	wake_up_q(&wake_q);
1562 out_free:
1563 	if (sem_io != fast_sem_io)
1564 		kvfree(sem_io);
1565 	return err;
1566 }
1567 
1568 static inline unsigned long
1569 copy_semid_from_user(struct semid64_ds *out, void __user *buf, int version)
1570 {
1571 	switch (version) {
1572 	case IPC_64:
1573 		if (copy_from_user(out, buf, sizeof(*out)))
1574 			return -EFAULT;
1575 		return 0;
1576 	case IPC_OLD:
1577 	    {
1578 		struct semid_ds tbuf_old;
1579 
1580 		if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
1581 			return -EFAULT;
1582 
1583 		out->sem_perm.uid	= tbuf_old.sem_perm.uid;
1584 		out->sem_perm.gid	= tbuf_old.sem_perm.gid;
1585 		out->sem_perm.mode	= tbuf_old.sem_perm.mode;
1586 
1587 		return 0;
1588 	    }
1589 	default:
1590 		return -EINVAL;
1591 	}
1592 }
1593 
1594 /*
1595  * This function handles some semctl commands which require the rwsem
1596  * to be held in write mode.
1597  * NOTE: no locks must be held, the rwsem is taken inside this function.
1598  */
1599 static int semctl_down(struct ipc_namespace *ns, int semid,
1600 		       int cmd, struct semid64_ds *semid64)
1601 {
1602 	struct sem_array *sma;
1603 	int err;
1604 	struct kern_ipc_perm *ipcp;
1605 
1606 	down_write(&sem_ids(ns).rwsem);
1607 	rcu_read_lock();
1608 
1609 	ipcp = ipcctl_obtain_check(ns, &sem_ids(ns), semid, cmd,
1610 				      &semid64->sem_perm, 0);
1611 	if (IS_ERR(ipcp)) {
1612 		err = PTR_ERR(ipcp);
1613 		goto out_unlock1;
1614 	}
1615 
1616 	sma = container_of(ipcp, struct sem_array, sem_perm);
1617 
1618 	err = security_sem_semctl(&sma->sem_perm, cmd);
1619 	if (err)
1620 		goto out_unlock1;
1621 
1622 	switch (cmd) {
1623 	case IPC_RMID:
1624 		sem_lock(sma, NULL, -1);
1625 		/* freeary unlocks the ipc object and rcu */
1626 		freeary(ns, ipcp);
1627 		goto out_up;
1628 	case IPC_SET:
1629 		sem_lock(sma, NULL, -1);
1630 		err = ipc_update_perm(&semid64->sem_perm, ipcp);
1631 		if (err)
1632 			goto out_unlock0;
1633 		sma->sem_ctime = ktime_get_real_seconds();
1634 		break;
1635 	default:
1636 		err = -EINVAL;
1637 		goto out_unlock1;
1638 	}
1639 
1640 out_unlock0:
1641 	sem_unlock(sma, -1);
1642 out_unlock1:
1643 	rcu_read_unlock();
1644 out_up:
1645 	up_write(&sem_ids(ns).rwsem);
1646 	return err;
1647 }
1648 
1649 static long ksys_semctl(int semid, int semnum, int cmd, unsigned long arg, int version)
1650 {
1651 	struct ipc_namespace *ns;
1652 	void __user *p = (void __user *)arg;
1653 	struct semid64_ds semid64;
1654 	int err;
1655 
1656 	if (semid < 0)
1657 		return -EINVAL;
1658 
1659 	ns = current->nsproxy->ipc_ns;
1660 
1661 	switch (cmd) {
1662 	case IPC_INFO:
1663 	case SEM_INFO:
1664 		return semctl_info(ns, semid, cmd, p);
1665 	case IPC_STAT:
1666 	case SEM_STAT:
1667 	case SEM_STAT_ANY:
1668 		err = semctl_stat(ns, semid, cmd, &semid64);
1669 		if (err < 0)
1670 			return err;
1671 		if (copy_semid_to_user(p, &semid64, version))
1672 			err = -EFAULT;
1673 		return err;
1674 	case GETALL:
1675 	case GETVAL:
1676 	case GETPID:
1677 	case GETNCNT:
1678 	case GETZCNT:
1679 	case SETALL:
1680 		return semctl_main(ns, semid, semnum, cmd, p);
1681 	case SETVAL: {
1682 		int val;
1683 #if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN)
1684 		/* big-endian 64bit */
1685 		val = arg >> 32;
1686 #else
1687 		/* 32bit or little-endian 64bit */
1688 		val = arg;
1689 #endif
1690 		return semctl_setval(ns, semid, semnum, val);
1691 	}
1692 	case IPC_SET:
1693 		if (copy_semid_from_user(&semid64, p, version))
1694 			return -EFAULT;
1695 		/* fall through */
1696 	case IPC_RMID:
1697 		return semctl_down(ns, semid, cmd, &semid64);
1698 	default:
1699 		return -EINVAL;
1700 	}
1701 }
1702 
1703 SYSCALL_DEFINE4(semctl, int, semid, int, semnum, int, cmd, unsigned long, arg)
1704 {
1705 	return ksys_semctl(semid, semnum, cmd, arg, IPC_64);
1706 }
1707 
1708 #ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
1709 long ksys_old_semctl(int semid, int semnum, int cmd, unsigned long arg)
1710 {
1711 	int version = ipc_parse_version(&cmd);
1712 
1713 	return ksys_semctl(semid, semnum, cmd, arg, version);
1714 }
1715 
1716 SYSCALL_DEFINE4(old_semctl, int, semid, int, semnum, int, cmd, unsigned long, arg)
1717 {
1718 	return ksys_old_semctl(semid, semnum, cmd, arg);
1719 }
1720 #endif
1721 
1722 #ifdef CONFIG_COMPAT
1723 
1724 struct compat_semid_ds {
1725 	struct compat_ipc_perm sem_perm;
1726 	old_time32_t sem_otime;
1727 	old_time32_t sem_ctime;
1728 	compat_uptr_t sem_base;
1729 	compat_uptr_t sem_pending;
1730 	compat_uptr_t sem_pending_last;
1731 	compat_uptr_t undo;
1732 	unsigned short sem_nsems;
1733 };
1734 
1735 static int copy_compat_semid_from_user(struct semid64_ds *out, void __user *buf,
1736 					int version)
1737 {
1738 	memset(out, 0, sizeof(*out));
1739 	if (version == IPC_64) {
1740 		struct compat_semid64_ds __user *p = buf;
1741 		return get_compat_ipc64_perm(&out->sem_perm, &p->sem_perm);
1742 	} else {
1743 		struct compat_semid_ds __user *p = buf;
1744 		return get_compat_ipc_perm(&out->sem_perm, &p->sem_perm);
1745 	}
1746 }
1747 
1748 static int copy_compat_semid_to_user(void __user *buf, struct semid64_ds *in,
1749 					int version)
1750 {
1751 	if (version == IPC_64) {
1752 		struct compat_semid64_ds v;
1753 		memset(&v, 0, sizeof(v));
1754 		to_compat_ipc64_perm(&v.sem_perm, &in->sem_perm);
1755 		v.sem_otime	 = lower_32_bits(in->sem_otime);
1756 		v.sem_otime_high = upper_32_bits(in->sem_otime);
1757 		v.sem_ctime	 = lower_32_bits(in->sem_ctime);
1758 		v.sem_ctime_high = upper_32_bits(in->sem_ctime);
1759 		v.sem_nsems = in->sem_nsems;
1760 		return copy_to_user(buf, &v, sizeof(v));
1761 	} else {
1762 		struct compat_semid_ds v;
1763 		memset(&v, 0, sizeof(v));
1764 		to_compat_ipc_perm(&v.sem_perm, &in->sem_perm);
1765 		v.sem_otime = in->sem_otime;
1766 		v.sem_ctime = in->sem_ctime;
1767 		v.sem_nsems = in->sem_nsems;
1768 		return copy_to_user(buf, &v, sizeof(v));
1769 	}
1770 }
1771 
1772 static long compat_ksys_semctl(int semid, int semnum, int cmd, int arg, int version)
1773 {
1774 	void __user *p = compat_ptr(arg);
1775 	struct ipc_namespace *ns;
1776 	struct semid64_ds semid64;
1777 	int err;
1778 
1779 	ns = current->nsproxy->ipc_ns;
1780 
1781 	if (semid < 0)
1782 		return -EINVAL;
1783 
1784 	switch (cmd & (~IPC_64)) {
1785 	case IPC_INFO:
1786 	case SEM_INFO:
1787 		return semctl_info(ns, semid, cmd, p);
1788 	case IPC_STAT:
1789 	case SEM_STAT:
1790 	case SEM_STAT_ANY:
1791 		err = semctl_stat(ns, semid, cmd, &semid64);
1792 		if (err < 0)
1793 			return err;
1794 		if (copy_compat_semid_to_user(p, &semid64, version))
1795 			err = -EFAULT;
1796 		return err;
1797 	case GETVAL:
1798 	case GETPID:
1799 	case GETNCNT:
1800 	case GETZCNT:
1801 	case GETALL:
1802 	case SETALL:
1803 		return semctl_main(ns, semid, semnum, cmd, p);
1804 	case SETVAL:
1805 		return semctl_setval(ns, semid, semnum, arg);
1806 	case IPC_SET:
1807 		if (copy_compat_semid_from_user(&semid64, p, version))
1808 			return -EFAULT;
1809 		/* fallthru */
1810 	case IPC_RMID:
1811 		return semctl_down(ns, semid, cmd, &semid64);
1812 	default:
1813 		return -EINVAL;
1814 	}
1815 }
1816 
1817 COMPAT_SYSCALL_DEFINE4(semctl, int, semid, int, semnum, int, cmd, int, arg)
1818 {
1819 	return compat_ksys_semctl(semid, semnum, cmd, arg, IPC_64);
1820 }
1821 
1822 #ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
1823 long compat_ksys_old_semctl(int semid, int semnum, int cmd, int arg)
1824 {
1825 	int version = compat_ipc_parse_version(&cmd);
1826 
1827 	return compat_ksys_semctl(semid, semnum, cmd, arg, version);
1828 }
1829 
1830 COMPAT_SYSCALL_DEFINE4(old_semctl, int, semid, int, semnum, int, cmd, int, arg)
1831 {
1832 	return compat_ksys_old_semctl(semid, semnum, cmd, arg);
1833 }
1834 #endif
1835 #endif
1836 
1837 /* If the task doesn't already have a undo_list, then allocate one
1838  * here.  We guarantee there is only one thread using this undo list,
1839  * and current is THE ONE
1840  *
1841  * If this allocation and assignment succeeds, but later
1842  * portions of this code fail, there is no need to free the sem_undo_list.
1843  * Just let it stay associated with the task, and it'll be freed later
1844  * at exit time.
1845  *
1846  * This can block, so callers must hold no locks.
1847  */
1848 static inline int get_undo_list(struct sem_undo_list **undo_listp)
1849 {
1850 	struct sem_undo_list *undo_list;
1851 
1852 	undo_list = current->sysvsem.undo_list;
1853 	if (!undo_list) {
1854 		undo_list = kzalloc(sizeof(*undo_list), GFP_KERNEL);
1855 		if (undo_list == NULL)
1856 			return -ENOMEM;
1857 		spin_lock_init(&undo_list->lock);
1858 		refcount_set(&undo_list->refcnt, 1);
1859 		INIT_LIST_HEAD(&undo_list->list_proc);
1860 
1861 		current->sysvsem.undo_list = undo_list;
1862 	}
1863 	*undo_listp = undo_list;
1864 	return 0;
1865 }
1866 
1867 static struct sem_undo *__lookup_undo(struct sem_undo_list *ulp, int semid)
1868 {
1869 	struct sem_undo *un;
1870 
1871 	list_for_each_entry_rcu(un, &ulp->list_proc, list_proc,
1872 				spin_is_locked(&ulp->lock)) {
1873 		if (un->semid == semid)
1874 			return un;
1875 	}
1876 	return NULL;
1877 }
1878 
1879 static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)
1880 {
1881 	struct sem_undo *un;
1882 
1883 	assert_spin_locked(&ulp->lock);
1884 
1885 	un = __lookup_undo(ulp, semid);
1886 	if (un) {
1887 		list_del_rcu(&un->list_proc);
1888 		list_add_rcu(&un->list_proc, &ulp->list_proc);
1889 	}
1890 	return un;
1891 }
1892 
1893 /**
1894  * find_alloc_undo - lookup (and if not present create) undo array
1895  * @ns: namespace
1896  * @semid: semaphore array id
1897  *
1898  * The function looks up (and if not present creates) the undo structure.
1899  * The size of the undo structure depends on the size of the semaphore
1900  * array, thus the alloc path is not that straightforward.
1901  * Lifetime-rules: sem_undo is rcu-protected, on success, the function
1902  * performs a rcu_read_lock().
1903  */
1904 static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
1905 {
1906 	struct sem_array *sma;
1907 	struct sem_undo_list *ulp;
1908 	struct sem_undo *un, *new;
1909 	int nsems, error;
1910 
1911 	error = get_undo_list(&ulp);
1912 	if (error)
1913 		return ERR_PTR(error);
1914 
1915 	rcu_read_lock();
1916 	spin_lock(&ulp->lock);
1917 	un = lookup_undo(ulp, semid);
1918 	spin_unlock(&ulp->lock);
1919 	if (likely(un != NULL))
1920 		goto out;
1921 
1922 	/* no undo structure around - allocate one. */
1923 	/* step 1: figure out the size of the semaphore array */
1924 	sma = sem_obtain_object_check(ns, semid);
1925 	if (IS_ERR(sma)) {
1926 		rcu_read_unlock();
1927 		return ERR_CAST(sma);
1928 	}
1929 
1930 	nsems = sma->sem_nsems;
1931 	if (!ipc_rcu_getref(&sma->sem_perm)) {
1932 		rcu_read_unlock();
1933 		un = ERR_PTR(-EIDRM);
1934 		goto out;
1935 	}
1936 	rcu_read_unlock();
1937 
1938 	/* step 2: allocate new undo structure */
1939 	new = kzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems, GFP_KERNEL);
1940 	if (!new) {
1941 		ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
1942 		return ERR_PTR(-ENOMEM);
1943 	}
1944 
1945 	/* step 3: Acquire the lock on semaphore array */
1946 	rcu_read_lock();
1947 	sem_lock_and_putref(sma);
1948 	if (!ipc_valid_object(&sma->sem_perm)) {
1949 		sem_unlock(sma, -1);
1950 		rcu_read_unlock();
1951 		kfree(new);
1952 		un = ERR_PTR(-EIDRM);
1953 		goto out;
1954 	}
1955 	spin_lock(&ulp->lock);
1956 
1957 	/*
1958 	 * step 4: check for races: did someone else allocate the undo struct?
1959 	 */
1960 	un = lookup_undo(ulp, semid);
1961 	if (un) {
1962 		kfree(new);
1963 		goto success;
1964 	}
1965 	/* step 5: initialize & link new undo structure */
1966 	new->semadj = (short *) &new[1];
1967 	new->ulp = ulp;
1968 	new->semid = semid;
1969 	assert_spin_locked(&ulp->lock);
1970 	list_add_rcu(&new->list_proc, &ulp->list_proc);
1971 	ipc_assert_locked_object(&sma->sem_perm);
1972 	list_add(&new->list_id, &sma->list_id);
1973 	un = new;
1974 
1975 success:
1976 	spin_unlock(&ulp->lock);
1977 	sem_unlock(sma, -1);
1978 out:
1979 	return un;
1980 }
1981 
1982 static long do_semtimedop(int semid, struct sembuf __user *tsops,
1983 		unsigned nsops, const struct timespec64 *timeout)
1984 {
1985 	int error = -EINVAL;
1986 	struct sem_array *sma;
1987 	struct sembuf fast_sops[SEMOPM_FAST];
1988 	struct sembuf *sops = fast_sops, *sop;
1989 	struct sem_undo *un;
1990 	int max, locknum;
1991 	bool undos = false, alter = false, dupsop = false;
1992 	struct sem_queue queue;
1993 	unsigned long dup = 0, jiffies_left = 0;
1994 	struct ipc_namespace *ns;
1995 
1996 	ns = current->nsproxy->ipc_ns;
1997 
1998 	if (nsops < 1 || semid < 0)
1999 		return -EINVAL;
2000 	if (nsops > ns->sc_semopm)
2001 		return -E2BIG;
2002 	if (nsops > SEMOPM_FAST) {
2003 		sops = kvmalloc_array(nsops, sizeof(*sops), GFP_KERNEL);
2004 		if (sops == NULL)
2005 			return -ENOMEM;
2006 	}
2007 
2008 	if (copy_from_user(sops, tsops, nsops * sizeof(*tsops))) {
2009 		error =  -EFAULT;
2010 		goto out_free;
2011 	}
2012 
2013 	if (timeout) {
2014 		if (timeout->tv_sec < 0 || timeout->tv_nsec < 0 ||
2015 			timeout->tv_nsec >= 1000000000L) {
2016 			error = -EINVAL;
2017 			goto out_free;
2018 		}
2019 		jiffies_left = timespec64_to_jiffies(timeout);
2020 	}
2021 
2022 	max = 0;
2023 	for (sop = sops; sop < sops + nsops; sop++) {
2024 		unsigned long mask = 1ULL << ((sop->sem_num) % BITS_PER_LONG);
2025 
2026 		if (sop->sem_num >= max)
2027 			max = sop->sem_num;
2028 		if (sop->sem_flg & SEM_UNDO)
2029 			undos = true;
2030 		if (dup & mask) {
2031 			/*
2032 			 * There was a previous alter access that appears
2033 			 * to have accessed the same semaphore, thus use
2034 			 * the dupsop logic. "appears", because the detection
2035 			 * can only check % BITS_PER_LONG.
2036 			 */
2037 			dupsop = true;
2038 		}
2039 		if (sop->sem_op != 0) {
2040 			alter = true;
2041 			dup |= mask;
2042 		}
2043 	}
2044 
2045 	if (undos) {
2046 		/* On success, find_alloc_undo takes the rcu_read_lock */
2047 		un = find_alloc_undo(ns, semid);
2048 		if (IS_ERR(un)) {
2049 			error = PTR_ERR(un);
2050 			goto out_free;
2051 		}
2052 	} else {
2053 		un = NULL;
2054 		rcu_read_lock();
2055 	}
2056 
2057 	sma = sem_obtain_object_check(ns, semid);
2058 	if (IS_ERR(sma)) {
2059 		rcu_read_unlock();
2060 		error = PTR_ERR(sma);
2061 		goto out_free;
2062 	}
2063 
2064 	error = -EFBIG;
2065 	if (max >= sma->sem_nsems) {
2066 		rcu_read_unlock();
2067 		goto out_free;
2068 	}
2069 
2070 	error = -EACCES;
2071 	if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) {
2072 		rcu_read_unlock();
2073 		goto out_free;
2074 	}
2075 
2076 	error = security_sem_semop(&sma->sem_perm, sops, nsops, alter);
2077 	if (error) {
2078 		rcu_read_unlock();
2079 		goto out_free;
2080 	}
2081 
2082 	error = -EIDRM;
2083 	locknum = sem_lock(sma, sops, nsops);
2084 	/*
2085 	 * We eventually might perform the following check in a lockless
2086 	 * fashion, considering ipc_valid_object() locking constraints.
2087 	 * If nsops == 1 and there is no contention for sem_perm.lock, then
2088 	 * only a per-semaphore lock is held and it's OK to proceed with the
2089 	 * check below. More details on the fine grained locking scheme
2090 	 * entangled here and why it's RMID race safe on comments at sem_lock()
2091 	 */
2092 	if (!ipc_valid_object(&sma->sem_perm))
2093 		goto out_unlock_free;
2094 	/*
2095 	 * semid identifiers are not unique - find_alloc_undo may have
2096 	 * allocated an undo structure, it was invalidated by an RMID
2097 	 * and now a new array with received the same id. Check and fail.
2098 	 * This case can be detected checking un->semid. The existence of
2099 	 * "un" itself is guaranteed by rcu.
2100 	 */
2101 	if (un && un->semid == -1)
2102 		goto out_unlock_free;
2103 
2104 	queue.sops = sops;
2105 	queue.nsops = nsops;
2106 	queue.undo = un;
2107 	queue.pid = task_tgid(current);
2108 	queue.alter = alter;
2109 	queue.dupsop = dupsop;
2110 
2111 	error = perform_atomic_semop(sma, &queue);
2112 	if (error == 0) { /* non-blocking succesfull path */
2113 		DEFINE_WAKE_Q(wake_q);
2114 
2115 		/*
2116 		 * If the operation was successful, then do
2117 		 * the required updates.
2118 		 */
2119 		if (alter)
2120 			do_smart_update(sma, sops, nsops, 1, &wake_q);
2121 		else
2122 			set_semotime(sma, sops);
2123 
2124 		sem_unlock(sma, locknum);
2125 		rcu_read_unlock();
2126 		wake_up_q(&wake_q);
2127 
2128 		goto out_free;
2129 	}
2130 	if (error < 0) /* non-blocking error path */
2131 		goto out_unlock_free;
2132 
2133 	/*
2134 	 * We need to sleep on this operation, so we put the current
2135 	 * task into the pending queue and go to sleep.
2136 	 */
2137 	if (nsops == 1) {
2138 		struct sem *curr;
2139 		int idx = array_index_nospec(sops->sem_num, sma->sem_nsems);
2140 		curr = &sma->sems[idx];
2141 
2142 		if (alter) {
2143 			if (sma->complex_count) {
2144 				list_add_tail(&queue.list,
2145 						&sma->pending_alter);
2146 			} else {
2147 
2148 				list_add_tail(&queue.list,
2149 						&curr->pending_alter);
2150 			}
2151 		} else {
2152 			list_add_tail(&queue.list, &curr->pending_const);
2153 		}
2154 	} else {
2155 		if (!sma->complex_count)
2156 			merge_queues(sma);
2157 
2158 		if (alter)
2159 			list_add_tail(&queue.list, &sma->pending_alter);
2160 		else
2161 			list_add_tail(&queue.list, &sma->pending_const);
2162 
2163 		sma->complex_count++;
2164 	}
2165 
2166 	do {
2167 		/* memory ordering ensured by the lock in sem_lock() */
2168 		WRITE_ONCE(queue.status, -EINTR);
2169 		queue.sleeper = current;
2170 
2171 		/* memory ordering is ensured by the lock in sem_lock() */
2172 		__set_current_state(TASK_INTERRUPTIBLE);
2173 		sem_unlock(sma, locknum);
2174 		rcu_read_unlock();
2175 
2176 		if (timeout)
2177 			jiffies_left = schedule_timeout(jiffies_left);
2178 		else
2179 			schedule();
2180 
2181 		/*
2182 		 * fastpath: the semop has completed, either successfully or
2183 		 * not, from the syscall pov, is quite irrelevant to us at this
2184 		 * point; we're done.
2185 		 *
2186 		 * We _do_ care, nonetheless, about being awoken by a signal or
2187 		 * spuriously.  The queue.status is checked again in the
2188 		 * slowpath (aka after taking sem_lock), such that we can detect
2189 		 * scenarios where we were awakened externally, during the
2190 		 * window between wake_q_add() and wake_up_q().
2191 		 */
2192 		error = READ_ONCE(queue.status);
2193 		if (error != -EINTR) {
2194 			/* see SEM_BARRIER_2 for purpose/pairing */
2195 			smp_acquire__after_ctrl_dep();
2196 			goto out_free;
2197 		}
2198 
2199 		rcu_read_lock();
2200 		locknum = sem_lock(sma, sops, nsops);
2201 
2202 		if (!ipc_valid_object(&sma->sem_perm))
2203 			goto out_unlock_free;
2204 
2205 		/*
2206 		 * No necessity for any barrier: We are protect by sem_lock()
2207 		 */
2208 		error = READ_ONCE(queue.status);
2209 
2210 		/*
2211 		 * If queue.status != -EINTR we are woken up by another process.
2212 		 * Leave without unlink_queue(), but with sem_unlock().
2213 		 */
2214 		if (error != -EINTR)
2215 			goto out_unlock_free;
2216 
2217 		/*
2218 		 * If an interrupt occurred we have to clean up the queue.
2219 		 */
2220 		if (timeout && jiffies_left == 0)
2221 			error = -EAGAIN;
2222 	} while (error == -EINTR && !signal_pending(current)); /* spurious */
2223 
2224 	unlink_queue(sma, &queue);
2225 
2226 out_unlock_free:
2227 	sem_unlock(sma, locknum);
2228 	rcu_read_unlock();
2229 out_free:
2230 	if (sops != fast_sops)
2231 		kvfree(sops);
2232 	return error;
2233 }
2234 
2235 long ksys_semtimedop(int semid, struct sembuf __user *tsops,
2236 		     unsigned int nsops, const struct __kernel_timespec __user *timeout)
2237 {
2238 	if (timeout) {
2239 		struct timespec64 ts;
2240 		if (get_timespec64(&ts, timeout))
2241 			return -EFAULT;
2242 		return do_semtimedop(semid, tsops, nsops, &ts);
2243 	}
2244 	return do_semtimedop(semid, tsops, nsops, NULL);
2245 }
2246 
2247 SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
2248 		unsigned int, nsops, const struct __kernel_timespec __user *, timeout)
2249 {
2250 	return ksys_semtimedop(semid, tsops, nsops, timeout);
2251 }
2252 
2253 #ifdef CONFIG_COMPAT_32BIT_TIME
2254 long compat_ksys_semtimedop(int semid, struct sembuf __user *tsems,
2255 			    unsigned int nsops,
2256 			    const struct old_timespec32 __user *timeout)
2257 {
2258 	if (timeout) {
2259 		struct timespec64 ts;
2260 		if (get_old_timespec32(&ts, timeout))
2261 			return -EFAULT;
2262 		return do_semtimedop(semid, tsems, nsops, &ts);
2263 	}
2264 	return do_semtimedop(semid, tsems, nsops, NULL);
2265 }
2266 
2267 SYSCALL_DEFINE4(semtimedop_time32, int, semid, struct sembuf __user *, tsems,
2268 		       unsigned int, nsops,
2269 		       const struct old_timespec32 __user *, timeout)
2270 {
2271 	return compat_ksys_semtimedop(semid, tsems, nsops, timeout);
2272 }
2273 #endif
2274 
2275 SYSCALL_DEFINE3(semop, int, semid, struct sembuf __user *, tsops,
2276 		unsigned, nsops)
2277 {
2278 	return do_semtimedop(semid, tsops, nsops, NULL);
2279 }
2280 
2281 /* If CLONE_SYSVSEM is set, establish sharing of SEM_UNDO state between
2282  * parent and child tasks.
2283  */
2284 
2285 int copy_semundo(unsigned long clone_flags, struct task_struct *tsk)
2286 {
2287 	struct sem_undo_list *undo_list;
2288 	int error;
2289 
2290 	if (clone_flags & CLONE_SYSVSEM) {
2291 		error = get_undo_list(&undo_list);
2292 		if (error)
2293 			return error;
2294 		refcount_inc(&undo_list->refcnt);
2295 		tsk->sysvsem.undo_list = undo_list;
2296 	} else
2297 		tsk->sysvsem.undo_list = NULL;
2298 
2299 	return 0;
2300 }
2301 
2302 /*
2303  * add semadj values to semaphores, free undo structures.
2304  * undo structures are not freed when semaphore arrays are destroyed
2305  * so some of them may be out of date.
2306  * IMPLEMENTATION NOTE: There is some confusion over whether the
2307  * set of adjustments that needs to be done should be done in an atomic
2308  * manner or not. That is, if we are attempting to decrement the semval
2309  * should we queue up and wait until we can do so legally?
2310  * The original implementation attempted to do this (queue and wait).
2311  * The current implementation does not do so. The POSIX standard
2312  * and SVID should be consulted to determine what behavior is mandated.
2313  */
2314 void exit_sem(struct task_struct *tsk)
2315 {
2316 	struct sem_undo_list *ulp;
2317 
2318 	ulp = tsk->sysvsem.undo_list;
2319 	if (!ulp)
2320 		return;
2321 	tsk->sysvsem.undo_list = NULL;
2322 
2323 	if (!refcount_dec_and_test(&ulp->refcnt))
2324 		return;
2325 
2326 	for (;;) {
2327 		struct sem_array *sma;
2328 		struct sem_undo *un;
2329 		int semid, i;
2330 		DEFINE_WAKE_Q(wake_q);
2331 
2332 		cond_resched();
2333 
2334 		rcu_read_lock();
2335 		un = list_entry_rcu(ulp->list_proc.next,
2336 				    struct sem_undo, list_proc);
2337 		if (&un->list_proc == &ulp->list_proc) {
2338 			/*
2339 			 * We must wait for freeary() before freeing this ulp,
2340 			 * in case we raced with last sem_undo. There is a small
2341 			 * possibility where we exit while freeary() didn't
2342 			 * finish unlocking sem_undo_list.
2343 			 */
2344 			spin_lock(&ulp->lock);
2345 			spin_unlock(&ulp->lock);
2346 			rcu_read_unlock();
2347 			break;
2348 		}
2349 		spin_lock(&ulp->lock);
2350 		semid = un->semid;
2351 		spin_unlock(&ulp->lock);
2352 
2353 		/* exit_sem raced with IPC_RMID, nothing to do */
2354 		if (semid == -1) {
2355 			rcu_read_unlock();
2356 			continue;
2357 		}
2358 
2359 		sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid);
2360 		/* exit_sem raced with IPC_RMID, nothing to do */
2361 		if (IS_ERR(sma)) {
2362 			rcu_read_unlock();
2363 			continue;
2364 		}
2365 
2366 		sem_lock(sma, NULL, -1);
2367 		/* exit_sem raced with IPC_RMID, nothing to do */
2368 		if (!ipc_valid_object(&sma->sem_perm)) {
2369 			sem_unlock(sma, -1);
2370 			rcu_read_unlock();
2371 			continue;
2372 		}
2373 		un = __lookup_undo(ulp, semid);
2374 		if (un == NULL) {
2375 			/* exit_sem raced with IPC_RMID+semget() that created
2376 			 * exactly the same semid. Nothing to do.
2377 			 */
2378 			sem_unlock(sma, -1);
2379 			rcu_read_unlock();
2380 			continue;
2381 		}
2382 
2383 		/* remove un from the linked lists */
2384 		ipc_assert_locked_object(&sma->sem_perm);
2385 		list_del(&un->list_id);
2386 
2387 		/* we are the last process using this ulp, acquiring ulp->lock
2388 		 * isn't required. Besides that, we are also protected against
2389 		 * IPC_RMID as we hold sma->sem_perm lock now
2390 		 */
2391 		list_del_rcu(&un->list_proc);
2392 
2393 		/* perform adjustments registered in un */
2394 		for (i = 0; i < sma->sem_nsems; i++) {
2395 			struct sem *semaphore = &sma->sems[i];
2396 			if (un->semadj[i]) {
2397 				semaphore->semval += un->semadj[i];
2398 				/*
2399 				 * Range checks of the new semaphore value,
2400 				 * not defined by sus:
2401 				 * - Some unices ignore the undo entirely
2402 				 *   (e.g. HP UX 11i 11.22, Tru64 V5.1)
2403 				 * - some cap the value (e.g. FreeBSD caps
2404 				 *   at 0, but doesn't enforce SEMVMX)
2405 				 *
2406 				 * Linux caps the semaphore value, both at 0
2407 				 * and at SEMVMX.
2408 				 *
2409 				 *	Manfred <manfred@colorfullife.com>
2410 				 */
2411 				if (semaphore->semval < 0)
2412 					semaphore->semval = 0;
2413 				if (semaphore->semval > SEMVMX)
2414 					semaphore->semval = SEMVMX;
2415 				ipc_update_pid(&semaphore->sempid, task_tgid(current));
2416 			}
2417 		}
2418 		/* maybe some queued-up processes were waiting for this */
2419 		do_smart_update(sma, NULL, 0, 1, &wake_q);
2420 		sem_unlock(sma, -1);
2421 		rcu_read_unlock();
2422 		wake_up_q(&wake_q);
2423 
2424 		kfree_rcu(un, rcu);
2425 	}
2426 	kfree(ulp);
2427 }
2428 
2429 #ifdef CONFIG_PROC_FS
2430 static int sysvipc_sem_proc_show(struct seq_file *s, void *it)
2431 {
2432 	struct user_namespace *user_ns = seq_user_ns(s);
2433 	struct kern_ipc_perm *ipcp = it;
2434 	struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
2435 	time64_t sem_otime;
2436 
2437 	/*
2438 	 * The proc interface isn't aware of sem_lock(), it calls
2439 	 * ipc_lock_object() directly (in sysvipc_find_ipc).
2440 	 * In order to stay compatible with sem_lock(), we must
2441 	 * enter / leave complex_mode.
2442 	 */
2443 	complexmode_enter(sma);
2444 
2445 	sem_otime = get_semotime(sma);
2446 
2447 	seq_printf(s,
2448 		   "%10d %10d  %4o %10u %5u %5u %5u %5u %10llu %10llu\n",
2449 		   sma->sem_perm.key,
2450 		   sma->sem_perm.id,
2451 		   sma->sem_perm.mode,
2452 		   sma->sem_nsems,
2453 		   from_kuid_munged(user_ns, sma->sem_perm.uid),
2454 		   from_kgid_munged(user_ns, sma->sem_perm.gid),
2455 		   from_kuid_munged(user_ns, sma->sem_perm.cuid),
2456 		   from_kgid_munged(user_ns, sma->sem_perm.cgid),
2457 		   sem_otime,
2458 		   sma->sem_ctime);
2459 
2460 	complexmode_tryleave(sma);
2461 
2462 	return 0;
2463 }
2464 #endif
2465