|
160
|
1 /* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
|
|
|
2 * Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
|
3 * of this software and associated documentation files (the "Software"), to
|
|
|
4 * deal in the Software without restriction, including without limitation the
|
|
|
5 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
|
6 * sell copies of the Software, and to permit persons to whom the Software is
|
|
|
7 * furnished to do so, subject to the following conditions:
|
|
|
8 *
|
|
|
9 * The above copyright notice and this permission notice shall be included in
|
|
|
10 * all copies or substantial portions of the Software.
|
|
|
11 *
|
|
|
12 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
13 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
14 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
|
15 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
16 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
17 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
18 * IN THE SOFTWARE.
|
|
|
19 */
|
|
|
20
|
|
|
21 /* We lean on the fact that POLL{IN,OUT,ERR,HUP} correspond with their
|
|
|
22 * EPOLL* counterparts. We use the POLL* variants in this file because that
|
|
|
23 * is what libuv uses elsewhere.
|
|
|
24 */
|
|
|
25
|
|
|
26 #include "uv.h"
|
|
|
27 #include "internal.h"
|
|
|
28
|
|
|
29 #include <inttypes.h>
|
|
|
30 #include <stdatomic.h>
|
|
|
31 #include <stddef.h> /* offsetof */
|
|
|
32 #include <stdint.h>
|
|
|
33 #include <stdio.h>
|
|
|
34 #include <stdlib.h>
|
|
|
35 #include <string.h>
|
|
|
36 #include <assert.h>
|
|
|
37 #include <errno.h>
|
|
|
38
|
|
|
39 #include <fcntl.h>
|
|
|
40 #include <ifaddrs.h>
|
|
|
41 #include <net/ethernet.h>
|
|
|
42 #include <net/if.h>
|
|
|
43 #include <netpacket/packet.h>
|
|
|
44 #include <sys/epoll.h>
|
|
|
45 #include <sys/inotify.h>
|
|
|
46 #include <sys/mman.h>
|
|
|
47 #include <sys/param.h>
|
|
|
48 #include <sys/prctl.h>
|
|
|
49 #include <sys/socket.h>
|
|
|
50 #include <sys/stat.h>
|
|
|
51 #include <sys/syscall.h>
|
|
|
52 #include <sys/sysinfo.h>
|
|
|
53 #include <sys/sysmacros.h>
|
|
|
54 #include <sys/types.h>
|
|
|
55 #include <sys/utsname.h>
|
|
|
56 #include <time.h>
|
|
|
57 #include <unistd.h>
|
|
|
58
|
|
|
59 #ifndef __NR_io_uring_setup
|
|
|
60 # define __NR_io_uring_setup 425
|
|
|
61 #endif
|
|
|
62
|
|
|
63 #ifndef __NR_io_uring_enter
|
|
|
64 # define __NR_io_uring_enter 426
|
|
|
65 #endif
|
|
|
66
|
|
|
67 #ifndef __NR_io_uring_register
|
|
|
68 # define __NR_io_uring_register 427
|
|
|
69 #endif
|
|
|
70
|
|
|
71 #ifndef __NR_copy_file_range
|
|
|
72 # if defined(__x86_64__)
|
|
|
73 # define __NR_copy_file_range 326
|
|
|
74 # elif defined(__i386__)
|
|
|
75 # define __NR_copy_file_range 377
|
|
|
76 # elif defined(__s390__)
|
|
|
77 # define __NR_copy_file_range 375
|
|
|
78 # elif defined(__arm__)
|
|
|
79 # define __NR_copy_file_range 391
|
|
|
80 # elif defined(__aarch64__)
|
|
|
81 # define __NR_copy_file_range 285
|
|
|
82 # elif defined(__powerpc__)
|
|
|
83 # define __NR_copy_file_range 379
|
|
|
84 # elif defined(__arc__)
|
|
|
85 # define __NR_copy_file_range 285
|
|
|
86 # elif defined(__riscv)
|
|
|
87 # define __NR_copy_file_range 285
|
|
|
88 # endif
|
|
|
89 #endif /* __NR_copy_file_range */
|
|
|
90
|
|
|
91 #ifndef __NR_statx
|
|
|
92 # if defined(__x86_64__)
|
|
|
93 # define __NR_statx 332
|
|
|
94 # elif defined(__i386__)
|
|
|
95 # define __NR_statx 383
|
|
|
96 # elif defined(__aarch64__)
|
|
|
97 # define __NR_statx 397
|
|
|
98 # elif defined(__arm__)
|
|
|
99 # define __NR_statx 397
|
|
|
100 # elif defined(__ppc__)
|
|
|
101 # define __NR_statx 383
|
|
|
102 # elif defined(__s390__)
|
|
|
103 # define __NR_statx 379
|
|
|
104 # elif defined(__riscv)
|
|
|
105 # define __NR_statx 291
|
|
|
106 # endif
|
|
|
107 #endif /* __NR_statx */
|
|
|
108
|
|
|
109 #ifndef __NR_getrandom
|
|
|
110 # if defined(__x86_64__)
|
|
|
111 # define __NR_getrandom 318
|
|
|
112 # elif defined(__i386__)
|
|
|
113 # define __NR_getrandom 355
|
|
|
114 # elif defined(__aarch64__)
|
|
|
115 # define __NR_getrandom 384
|
|
|
116 # elif defined(__arm__)
|
|
|
117 # define __NR_getrandom 384
|
|
|
118 # elif defined(__ppc__)
|
|
|
119 # define __NR_getrandom 359
|
|
|
120 # elif defined(__s390__)
|
|
|
121 # define __NR_getrandom 349
|
|
|
122 # elif defined(__riscv)
|
|
|
123 # define __NR_getrandom 278
|
|
|
124 # endif
|
|
|
125 #endif /* __NR_getrandom */
|
|
|
126
|
|
|
127 enum {
|
|
|
128 UV__IORING_SETUP_SQPOLL = 2u,
|
|
|
129 UV__IORING_SETUP_NO_SQARRAY = 0x10000u,
|
|
|
130 };
|
|
|
131
|
|
|
132 enum {
|
|
|
133 UV__IORING_FEAT_SINGLE_MMAP = 1u,
|
|
|
134 UV__IORING_FEAT_NODROP = 2u,
|
|
|
135 UV__IORING_FEAT_RSRC_TAGS = 1024u, /* linux v5.13 */
|
|
|
136 };
|
|
|
137
|
|
|
138 enum {
|
|
|
139 UV__IORING_OP_READV = 1,
|
|
|
140 UV__IORING_OP_WRITEV = 2,
|
|
|
141 UV__IORING_OP_FSYNC = 3,
|
|
|
142 UV__IORING_OP_OPENAT = 18,
|
|
|
143 UV__IORING_OP_CLOSE = 19,
|
|
|
144 UV__IORING_OP_STATX = 21,
|
|
|
145 UV__IORING_OP_EPOLL_CTL = 29,
|
|
|
146 UV__IORING_OP_RENAMEAT = 35,
|
|
|
147 UV__IORING_OP_UNLINKAT = 36,
|
|
|
148 UV__IORING_OP_MKDIRAT = 37,
|
|
|
149 UV__IORING_OP_SYMLINKAT = 38,
|
|
|
150 UV__IORING_OP_LINKAT = 39,
|
|
|
151 UV__IORING_OP_FTRUNCATE = 55,
|
|
|
152 };
|
|
|
153
|
|
|
154 enum {
|
|
|
155 UV__IORING_ENTER_GETEVENTS = 1u,
|
|
|
156 UV__IORING_ENTER_SQ_WAKEUP = 2u,
|
|
|
157 };
|
|
|
158
|
|
|
159 enum {
|
|
|
160 UV__IORING_SQ_NEED_WAKEUP = 1u,
|
|
|
161 UV__IORING_SQ_CQ_OVERFLOW = 2u,
|
|
|
162 };
|
|
|
163
|
|
|
164 struct uv__io_cqring_offsets {
|
|
|
165 uint32_t head;
|
|
|
166 uint32_t tail;
|
|
|
167 uint32_t ring_mask;
|
|
|
168 uint32_t ring_entries;
|
|
|
169 uint32_t overflow;
|
|
|
170 uint32_t cqes;
|
|
|
171 uint64_t reserved0;
|
|
|
172 uint64_t reserved1;
|
|
|
173 };
|
|
|
174
|
|
|
175 STATIC_ASSERT(40 == sizeof(struct uv__io_cqring_offsets));
|
|
|
176
|
|
|
177 struct uv__io_sqring_offsets {
|
|
|
178 uint32_t head;
|
|
|
179 uint32_t tail;
|
|
|
180 uint32_t ring_mask;
|
|
|
181 uint32_t ring_entries;
|
|
|
182 uint32_t flags;
|
|
|
183 uint32_t dropped;
|
|
|
184 uint32_t array;
|
|
|
185 uint32_t reserved0;
|
|
|
186 uint64_t reserved1;
|
|
|
187 };
|
|
|
188
|
|
|
189 STATIC_ASSERT(40 == sizeof(struct uv__io_sqring_offsets));
|
|
|
190
|
|
|
191 struct uv__io_uring_cqe {
|
|
|
192 uint64_t user_data;
|
|
|
193 int32_t res;
|
|
|
194 uint32_t flags;
|
|
|
195 };
|
|
|
196
|
|
|
197 STATIC_ASSERT(16 == sizeof(struct uv__io_uring_cqe));
|
|
|
198
|
|
|
199 struct uv__io_uring_sqe {
|
|
|
200 uint8_t opcode;
|
|
|
201 uint8_t flags;
|
|
|
202 uint16_t ioprio;
|
|
|
203 int32_t fd;
|
|
|
204 union {
|
|
|
205 uint64_t off;
|
|
|
206 uint64_t addr2;
|
|
|
207 };
|
|
|
208 union {
|
|
|
209 uint64_t addr;
|
|
|
210 };
|
|
|
211 uint32_t len;
|
|
|
212 union {
|
|
|
213 uint32_t rw_flags;
|
|
|
214 uint32_t fsync_flags;
|
|
|
215 uint32_t open_flags;
|
|
|
216 uint32_t statx_flags;
|
|
|
217 };
|
|
|
218 uint64_t user_data;
|
|
|
219 union {
|
|
|
220 uint16_t buf_index;
|
|
|
221 uint64_t pad[3];
|
|
|
222 };
|
|
|
223 };
|
|
|
224
|
|
|
225 STATIC_ASSERT(64 == sizeof(struct uv__io_uring_sqe));
|
|
|
226 STATIC_ASSERT(0 == offsetof(struct uv__io_uring_sqe, opcode));
|
|
|
227 STATIC_ASSERT(1 == offsetof(struct uv__io_uring_sqe, flags));
|
|
|
228 STATIC_ASSERT(2 == offsetof(struct uv__io_uring_sqe, ioprio));
|
|
|
229 STATIC_ASSERT(4 == offsetof(struct uv__io_uring_sqe, fd));
|
|
|
230 STATIC_ASSERT(8 == offsetof(struct uv__io_uring_sqe, off));
|
|
|
231 STATIC_ASSERT(16 == offsetof(struct uv__io_uring_sqe, addr));
|
|
|
232 STATIC_ASSERT(24 == offsetof(struct uv__io_uring_sqe, len));
|
|
|
233 STATIC_ASSERT(28 == offsetof(struct uv__io_uring_sqe, rw_flags));
|
|
|
234 STATIC_ASSERT(32 == offsetof(struct uv__io_uring_sqe, user_data));
|
|
|
235 STATIC_ASSERT(40 == offsetof(struct uv__io_uring_sqe, buf_index));
|
|
|
236
|
|
|
237 struct uv__io_uring_params {
|
|
|
238 uint32_t sq_entries;
|
|
|
239 uint32_t cq_entries;
|
|
|
240 uint32_t flags;
|
|
|
241 uint32_t sq_thread_cpu;
|
|
|
242 uint32_t sq_thread_idle;
|
|
|
243 uint32_t features;
|
|
|
244 uint32_t reserved[4];
|
|
|
245 struct uv__io_sqring_offsets sq_off; /* 40 bytes */
|
|
|
246 struct uv__io_cqring_offsets cq_off; /* 40 bytes */
|
|
|
247 };
|
|
|
248
|
|
|
249 STATIC_ASSERT(40 + 40 + 40 == sizeof(struct uv__io_uring_params));
|
|
|
250 STATIC_ASSERT(40 == offsetof(struct uv__io_uring_params, sq_off));
|
|
|
251 STATIC_ASSERT(80 == offsetof(struct uv__io_uring_params, cq_off));
|
|
|
252
|
|
|
253 STATIC_ASSERT(EPOLL_CTL_ADD < 4);
|
|
|
254 STATIC_ASSERT(EPOLL_CTL_DEL < 4);
|
|
|
255 STATIC_ASSERT(EPOLL_CTL_MOD < 4);
|
|
|
256
|
|
|
257 struct watcher_list {
|
|
|
258 RB_ENTRY(watcher_list) entry;
|
|
|
259 struct uv__queue watchers;
|
|
|
260 int iterating;
|
|
|
261 char* path;
|
|
|
262 int wd;
|
|
|
263 };
|
|
|
264
|
|
|
265 struct watcher_root {
|
|
|
266 struct watcher_list* rbh_root;
|
|
|
267 };
|
|
|
268
|
|
|
269 static int uv__inotify_fork(uv_loop_t* loop, struct watcher_list* root);
|
|
|
270 static void uv__inotify_read(uv_loop_t* loop,
|
|
|
271 uv__io_t* w,
|
|
|
272 unsigned int revents);
|
|
|
273 static int compare_watchers(const struct watcher_list* a,
|
|
|
274 const struct watcher_list* b);
|
|
|
275 static void maybe_free_watcher_list(struct watcher_list* w,
|
|
|
276 uv_loop_t* loop);
|
|
|
277
|
|
|
278 static void uv__epoll_ctl_flush(int epollfd,
|
|
|
279 struct uv__iou* ctl,
|
|
|
280 struct epoll_event (*events)[256]);
|
|
|
281
|
|
|
282 static void uv__epoll_ctl_prep(int epollfd,
|
|
|
283 struct uv__iou* ctl,
|
|
|
284 struct epoll_event (*events)[256],
|
|
|
285 int op,
|
|
|
286 int fd,
|
|
|
287 struct epoll_event* e);
|
|
|
288
|
|
|
289 RB_GENERATE_STATIC(watcher_root, watcher_list, entry, compare_watchers)
|
|
|
290
|
|
|
291
|
|
|
292 static struct watcher_root* uv__inotify_watchers(uv_loop_t* loop) {
|
|
|
293 /* This cast works because watcher_root is a struct with a pointer as its
|
|
|
294 * sole member. Such type punning is unsafe in the presence of strict
|
|
|
295 * pointer aliasing (and is just plain nasty) but that is why libuv
|
|
|
296 * is compiled with -fno-strict-aliasing.
|
|
|
297 */
|
|
|
298 return (struct watcher_root*) &loop->inotify_watchers;
|
|
|
299 }
|
|
|
300
|
|
|
301
|
|
|
302 unsigned uv__kernel_version(void) {
|
|
|
303 static _Atomic unsigned cached_version;
|
|
|
304 struct utsname u;
|
|
|
305 unsigned version;
|
|
|
306 unsigned major;
|
|
|
307 unsigned minor;
|
|
|
308 unsigned patch;
|
|
|
309 char v_sig[256];
|
|
|
310 char* needle;
|
|
|
311
|
|
|
312 version = atomic_load_explicit(&cached_version, memory_order_relaxed);
|
|
|
313 if (version != 0)
|
|
|
314 return version;
|
|
|
315
|
|
|
316 /* Check /proc/version_signature first as it's the way to get the mainline
|
|
|
317 * kernel version in Ubuntu. The format is:
|
|
|
318 * Ubuntu ubuntu_kernel_version mainline_kernel_version
|
|
|
319 * For example:
|
|
|
320 * Ubuntu 5.15.0-79.86-generic 5.15.111
|
|
|
321 */
|
|
|
322 if (0 == uv__slurp("/proc/version_signature", v_sig, sizeof(v_sig)))
|
|
|
323 if (3 == sscanf(v_sig, "Ubuntu %*s %u.%u.%u", &major, &minor, &patch))
|
|
|
324 goto calculate_version;
|
|
|
325
|
|
|
326 if (-1 == uname(&u))
|
|
|
327 return 0;
|
|
|
328
|
|
|
329 /* In Debian we need to check `version` instead of `release` to extract the
|
|
|
330 * mainline kernel version. This is an example of how it looks like:
|
|
|
331 * #1 SMP Debian 5.10.46-4 (2021-08-03)
|
|
|
332 */
|
|
|
333 needle = strstr(u.version, "Debian ");
|
|
|
334 if (needle != NULL)
|
|
|
335 if (3 == sscanf(needle, "Debian %u.%u.%u", &major, &minor, &patch))
|
|
|
336 goto calculate_version;
|
|
|
337
|
|
|
338 if (3 != sscanf(u.release, "%u.%u.%u", &major, &minor, &patch))
|
|
|
339 return 0;
|
|
|
340
|
|
|
341 /* Handle it when the process runs under the UNAME26 personality:
|
|
|
342 *
|
|
|
343 * - kernels >= 3.x identify as 2.6.40+x
|
|
|
344 * - kernels >= 4.x identify as 2.6.60+x
|
|
|
345 *
|
|
|
346 * UNAME26 is a poorly conceived hack that doesn't let us distinguish
|
|
|
347 * between 4.x kernels and 5.x/6.x kernels so we conservatively assume
|
|
|
348 * that 2.6.60+x means 4.x.
|
|
|
349 *
|
|
|
350 * Fun fact of the day: it's technically possible to observe the actual
|
|
|
351 * kernel version for a brief moment because uname() first copies out the
|
|
|
352 * real release string before overwriting it with the backcompat string.
|
|
|
353 */
|
|
|
354 if (major == 2 && minor == 6) {
|
|
|
355 if (patch >= 60) {
|
|
|
356 major = 4;
|
|
|
357 minor = patch - 60;
|
|
|
358 patch = 0;
|
|
|
359 } else if (patch >= 40) {
|
|
|
360 major = 3;
|
|
|
361 minor = patch - 40;
|
|
|
362 patch = 0;
|
|
|
363 }
|
|
|
364 }
|
|
|
365
|
|
|
366 calculate_version:
|
|
|
367 version = major * 65536 + minor * 256 + patch;
|
|
|
368 atomic_store_explicit(&cached_version, version, memory_order_relaxed);
|
|
|
369
|
|
|
370 return version;
|
|
|
371 }
|
|
|
372
|
|
|
373
|
|
|
374 ssize_t
|
|
|
375 uv__fs_copy_file_range(int fd_in,
|
|
|
376 off_t* off_in,
|
|
|
377 int fd_out,
|
|
|
378 off_t* off_out,
|
|
|
379 size_t len,
|
|
|
380 unsigned int flags)
|
|
|
381 {
|
|
|
382 #ifdef __NR_copy_file_range
|
|
|
383 return syscall(__NR_copy_file_range,
|
|
|
384 fd_in,
|
|
|
385 off_in,
|
|
|
386 fd_out,
|
|
|
387 off_out,
|
|
|
388 len,
|
|
|
389 flags);
|
|
|
390 #else
|
|
|
391 return errno = ENOSYS, -1;
|
|
|
392 #endif
|
|
|
393 }
|
|
|
394
|
|
|
395
|
|
|
396 int uv__statx(int dirfd,
|
|
|
397 const char* path,
|
|
|
398 int flags,
|
|
|
399 unsigned int mask,
|
|
|
400 struct uv__statx* statxbuf) {
|
|
|
401 #if !defined(__NR_statx) || defined(__ANDROID_API__) && __ANDROID_API__ < 30
|
|
|
402 return errno = ENOSYS, -1;
|
|
|
403 #else
|
|
|
404 int rc;
|
|
|
405
|
|
|
406 rc = syscall(__NR_statx, dirfd, path, flags, mask, statxbuf);
|
|
|
407 if (rc >= 0)
|
|
|
408 uv__msan_unpoison(statxbuf, sizeof(*statxbuf));
|
|
|
409
|
|
|
410 return rc;
|
|
|
411 #endif
|
|
|
412 }
|
|
|
413
|
|
|
414
|
|
|
415 ssize_t uv__getrandom(void* buf, size_t buflen, unsigned flags) {
|
|
|
416 #if !defined(__NR_getrandom) || defined(__ANDROID_API__) && __ANDROID_API__ < 28
|
|
|
417 return errno = ENOSYS, -1;
|
|
|
418 #else
|
|
|
419 ssize_t rc;
|
|
|
420
|
|
|
421 rc = syscall(__NR_getrandom, buf, buflen, flags);
|
|
|
422 if (rc >= 0)
|
|
|
423 uv__msan_unpoison(buf, buflen);
|
|
|
424
|
|
|
425 return rc;
|
|
|
426 #endif
|
|
|
427 }
|
|
|
428
|
|
|
429
|
|
|
430 int uv__io_uring_setup(int entries, struct uv__io_uring_params* params) {
|
|
|
431 return syscall(__NR_io_uring_setup, entries, params);
|
|
|
432 }
|
|
|
433
|
|
|
434
|
|
|
435 int uv__io_uring_enter(int fd,
|
|
|
436 unsigned to_submit,
|
|
|
437 unsigned min_complete,
|
|
|
438 unsigned flags) {
|
|
|
439 /* io_uring_enter used to take a sigset_t but it's unused
|
|
|
440 * in newer kernels unless IORING_ENTER_EXT_ARG is set,
|
|
|
441 * in which case it takes a struct io_uring_getevents_arg.
|
|
|
442 */
|
|
|
443 return syscall(__NR_io_uring_enter,
|
|
|
444 fd,
|
|
|
445 to_submit,
|
|
|
446 min_complete,
|
|
|
447 flags,
|
|
|
448 NULL,
|
|
|
449 0L);
|
|
|
450 }
|
|
|
451
|
|
|
452
|
|
|
453 int uv__io_uring_register(int fd, unsigned opcode, void* arg, unsigned nargs) {
|
|
|
454 return syscall(__NR_io_uring_register, fd, opcode, arg, nargs);
|
|
|
455 }
|
|
|
456
|
|
|
457
|
|
|
458 static int uv__use_io_uring(uint32_t flags) {
|
|
|
459 #if defined(__ANDROID_API__)
|
|
|
460 return 0; /* Possibly available but blocked by seccomp. */
|
|
|
461 #elif defined(__arm__) && __SIZEOF_POINTER__ == 4
|
|
|
462 /* See https://github.com/libuv/libuv/issues/4158. */
|
|
|
463 return 0; /* All 32 bits kernels appear buggy. */
|
|
|
464 #elif defined(__powerpc64__) || defined(__ppc64__)
|
|
|
465 /* See https://github.com/libuv/libuv/issues/4283. */
|
|
|
466 return 0; /* Random SIGSEGV in signal handler. */
|
|
|
467 #else
|
|
|
468 /* Ternary: unknown=0, yes=1, no=-1 */
|
|
|
469 static _Atomic int use_io_uring;
|
|
|
470 char* val;
|
|
|
471 int use;
|
|
|
472
|
|
|
473 #if defined(__hppa__)
|
|
|
474 /* io_uring first supported on parisc in 6.1, functional in .51
|
|
|
475 * https://lore.kernel.org/all/[email protected]/
|
|
|
476 */
|
|
|
477 if (uv__kernel_version() < /*6.1.51*/0x060133)
|
|
|
478 return 0;
|
|
|
479 #endif
|
|
|
480
|
|
|
481 /* SQPOLL is all kinds of buggy but epoll batching should work fine. */
|
|
|
482 if (0 == (flags & UV__IORING_SETUP_SQPOLL))
|
|
|
483 return 1;
|
|
|
484
|
|
|
485 /* Older kernels have a bug where the sqpoll thread uses 100% CPU. */
|
|
|
486 if (uv__kernel_version() < /*5.10.186*/0x050ABA)
|
|
|
487 return 0;
|
|
|
488
|
|
|
489 use = atomic_load_explicit(&use_io_uring, memory_order_relaxed);
|
|
|
490
|
|
|
491 if (use == 0) {
|
|
|
492 val = getenv("UV_USE_IO_URING");
|
|
|
493 use = val != NULL && atoi(val) > 0 ? 1 : -1;
|
|
|
494 atomic_store_explicit(&use_io_uring, use, memory_order_relaxed);
|
|
|
495 }
|
|
|
496
|
|
|
497 return use > 0;
|
|
|
498 #endif
|
|
|
499 }
|
|
|
500
|
|
|
501
|
|
|
502 static void uv__iou_init(int epollfd,
|
|
|
503 struct uv__iou* iou,
|
|
|
504 uint32_t entries,
|
|
|
505 uint32_t flags) {
|
|
|
506 struct uv__io_uring_params params;
|
|
|
507 struct epoll_event e;
|
|
|
508 size_t cqlen;
|
|
|
509 size_t sqlen;
|
|
|
510 size_t maxlen;
|
|
|
511 size_t sqelen;
|
|
|
512 unsigned kernel_version;
|
|
|
513 uint32_t* sqarray;
|
|
|
514 uint32_t i;
|
|
|
515 char* sq;
|
|
|
516 char* sqe;
|
|
|
517 int ringfd;
|
|
|
518 int no_sqarray;
|
|
|
519
|
|
|
520 sq = MAP_FAILED;
|
|
|
521 sqe = MAP_FAILED;
|
|
|
522
|
|
|
523 if (!uv__use_io_uring(flags))
|
|
|
524 return;
|
|
|
525
|
|
|
526 kernel_version = uv__kernel_version();
|
|
|
527 no_sqarray =
|
|
|
528 UV__IORING_SETUP_NO_SQARRAY * (kernel_version >= /* 6.6 */0x060600);
|
|
|
529
|
|
|
530 /* SQPOLL required CAP_SYS_NICE until linux v5.12 relaxed that requirement.
|
|
|
531 * Mostly academic because we check for a v5.13 kernel afterwards anyway.
|
|
|
532 */
|
|
|
533 memset(¶ms, 0, sizeof(params));
|
|
|
534 params.flags = flags | no_sqarray;
|
|
|
535
|
|
|
536 if (flags & UV__IORING_SETUP_SQPOLL)
|
|
|
537 params.sq_thread_idle = 10; /* milliseconds */
|
|
|
538
|
|
|
539 /* Kernel returns a file descriptor with O_CLOEXEC flag set. */
|
|
|
540 ringfd = uv__io_uring_setup(entries, ¶ms);
|
|
|
541 if (ringfd == -1)
|
|
|
542 return;
|
|
|
543
|
|
|
544 /* IORING_FEAT_RSRC_TAGS is used to detect linux v5.13 but what we're
|
|
|
545 * actually detecting is whether IORING_OP_STATX works with SQPOLL.
|
|
|
546 */
|
|
|
547 if (!(params.features & UV__IORING_FEAT_RSRC_TAGS))
|
|
|
548 goto fail;
|
|
|
549
|
|
|
550 /* Implied by IORING_FEAT_RSRC_TAGS but checked explicitly anyway. */
|
|
|
551 if (!(params.features & UV__IORING_FEAT_SINGLE_MMAP))
|
|
|
552 goto fail;
|
|
|
553
|
|
|
554 /* Implied by IORING_FEAT_RSRC_TAGS but checked explicitly anyway. */
|
|
|
555 if (!(params.features & UV__IORING_FEAT_NODROP))
|
|
|
556 goto fail;
|
|
|
557
|
|
|
558 sqlen = params.sq_off.array + params.sq_entries * sizeof(uint32_t);
|
|
|
559 cqlen =
|
|
|
560 params.cq_off.cqes + params.cq_entries * sizeof(struct uv__io_uring_cqe);
|
|
|
561 maxlen = sqlen < cqlen ? cqlen : sqlen;
|
|
|
562 sqelen = params.sq_entries * sizeof(struct uv__io_uring_sqe);
|
|
|
563
|
|
|
564 sq = mmap(0,
|
|
|
565 maxlen,
|
|
|
566 PROT_READ | PROT_WRITE,
|
|
|
567 MAP_SHARED | MAP_POPULATE,
|
|
|
568 ringfd,
|
|
|
569 0); /* IORING_OFF_SQ_RING */
|
|
|
570
|
|
|
571 sqe = mmap(0,
|
|
|
572 sqelen,
|
|
|
573 PROT_READ | PROT_WRITE,
|
|
|
574 MAP_SHARED | MAP_POPULATE,
|
|
|
575 ringfd,
|
|
|
576 0x10000000ull); /* IORING_OFF_SQES */
|
|
|
577
|
|
|
578 if (sq == MAP_FAILED || sqe == MAP_FAILED)
|
|
|
579 goto fail;
|
|
|
580
|
|
|
581 if (flags & UV__IORING_SETUP_SQPOLL) {
|
|
|
582 /* Only interested in completion events. To get notified when
|
|
|
583 * the kernel pulls items from the submission ring, add POLLOUT.
|
|
|
584 */
|
|
|
585 memset(&e, 0, sizeof(e));
|
|
|
586 e.events = POLLIN;
|
|
|
587 e.data.fd = ringfd;
|
|
|
588
|
|
|
589 if (epoll_ctl(epollfd, EPOLL_CTL_ADD, ringfd, &e))
|
|
|
590 goto fail;
|
|
|
591 }
|
|
|
592
|
|
|
593 iou->sqhead = (uint32_t*) (sq + params.sq_off.head);
|
|
|
594 iou->sqtail = (uint32_t*) (sq + params.sq_off.tail);
|
|
|
595 iou->sqmask = *(uint32_t*) (sq + params.sq_off.ring_mask);
|
|
|
596 iou->sqflags = (uint32_t*) (sq + params.sq_off.flags);
|
|
|
597 iou->cqhead = (uint32_t*) (sq + params.cq_off.head);
|
|
|
598 iou->cqtail = (uint32_t*) (sq + params.cq_off.tail);
|
|
|
599 iou->cqmask = *(uint32_t*) (sq + params.cq_off.ring_mask);
|
|
|
600 iou->sq = sq;
|
|
|
601 iou->cqe = sq + params.cq_off.cqes;
|
|
|
602 iou->sqe = sqe;
|
|
|
603 iou->sqlen = sqlen;
|
|
|
604 iou->cqlen = cqlen;
|
|
|
605 iou->maxlen = maxlen;
|
|
|
606 iou->sqelen = sqelen;
|
|
|
607 iou->ringfd = ringfd;
|
|
|
608 iou->in_flight = 0;
|
|
|
609
|
|
|
610 if (no_sqarray)
|
|
|
611 return;
|
|
|
612
|
|
|
613 sqarray = (uint32_t*) (sq + params.sq_off.array);
|
|
|
614 for (i = 0; i <= iou->sqmask; i++)
|
|
|
615 sqarray[i] = i; /* Slot -> sqe identity mapping. */
|
|
|
616
|
|
|
617 return;
|
|
|
618
|
|
|
619 fail:
|
|
|
620 if (sq != MAP_FAILED)
|
|
|
621 munmap(sq, maxlen);
|
|
|
622
|
|
|
623 if (sqe != MAP_FAILED)
|
|
|
624 munmap(sqe, sqelen);
|
|
|
625
|
|
|
626 uv__close(ringfd);
|
|
|
627 }
|
|
|
628
|
|
|
629
|
|
|
630 static void uv__iou_delete(struct uv__iou* iou) {
|
|
|
631 if (iou->ringfd > -1) {
|
|
|
632 munmap(iou->sq, iou->maxlen);
|
|
|
633 munmap(iou->sqe, iou->sqelen);
|
|
|
634 uv__close(iou->ringfd);
|
|
|
635 iou->ringfd = -1;
|
|
|
636 }
|
|
|
637 }
|
|
|
638
|
|
|
639
|
|
|
640 int uv__platform_loop_init(uv_loop_t* loop) {
|
|
|
641 uv__loop_internal_fields_t* lfields;
|
|
|
642
|
|
|
643 lfields = uv__get_internal_fields(loop);
|
|
|
644 lfields->ctl.ringfd = -1;
|
|
|
645 lfields->iou.ringfd = -2; /* "uninitialized" */
|
|
|
646
|
|
|
647 loop->inotify_watchers = NULL;
|
|
|
648 loop->inotify_fd = -1;
|
|
|
649 loop->backend_fd = epoll_create1(O_CLOEXEC);
|
|
|
650
|
|
|
651 if (loop->backend_fd == -1)
|
|
|
652 return UV__ERR(errno);
|
|
|
653
|
|
|
654 uv__iou_init(loop->backend_fd, &lfields->ctl, 256, 0);
|
|
|
655
|
|
|
656 return 0;
|
|
|
657 }
|
|
|
658
|
|
|
659
|
|
|
660 int uv__io_fork(uv_loop_t* loop) {
|
|
|
661 int err;
|
|
|
662 struct watcher_list* root;
|
|
|
663
|
|
|
664 root = uv__inotify_watchers(loop)->rbh_root;
|
|
|
665
|
|
|
666 uv__close(loop->backend_fd);
|
|
|
667 loop->backend_fd = -1;
|
|
|
668
|
|
|
669 /* TODO(bnoordhuis) Loses items from the submission and completion rings. */
|
|
|
670 uv__platform_loop_delete(loop);
|
|
|
671
|
|
|
672 err = uv__platform_loop_init(loop);
|
|
|
673 if (err)
|
|
|
674 return err;
|
|
|
675
|
|
|
676 return uv__inotify_fork(loop, root);
|
|
|
677 }
|
|
|
678
|
|
|
679
|
|
|
680 void uv__platform_loop_delete(uv_loop_t* loop) {
|
|
|
681 uv__loop_internal_fields_t* lfields;
|
|
|
682
|
|
|
683 lfields = uv__get_internal_fields(loop);
|
|
|
684 uv__iou_delete(&lfields->ctl);
|
|
|
685 uv__iou_delete(&lfields->iou);
|
|
|
686
|
|
|
687 if (loop->inotify_fd != -1) {
|
|
|
688 uv__io_stop(loop, &loop->inotify_read_watcher, POLLIN);
|
|
|
689 uv__close(loop->inotify_fd);
|
|
|
690 loop->inotify_fd = -1;
|
|
|
691 }
|
|
|
692 }
|
|
|
693
|
|
|
694
|
|
|
695 struct uv__invalidate {
|
|
|
696 struct epoll_event (*prep)[256];
|
|
|
697 struct epoll_event* events;
|
|
|
698 int nfds;
|
|
|
699 };
|
|
|
700
|
|
|
701
|
|
|
702 void uv__platform_invalidate_fd(uv_loop_t* loop, int fd) {
|
|
|
703 uv__loop_internal_fields_t* lfields;
|
|
|
704 struct uv__invalidate* inv;
|
|
|
705 struct epoll_event dummy;
|
|
|
706 int i;
|
|
|
707
|
|
|
708 lfields = uv__get_internal_fields(loop);
|
|
|
709 inv = lfields->inv;
|
|
|
710
|
|
|
711 /* Invalidate events with same file descriptor */
|
|
|
712 if (inv != NULL)
|
|
|
713 for (i = 0; i < inv->nfds; i++)
|
|
|
714 if (inv->events[i].data.fd == fd)
|
|
|
715 inv->events[i].data.fd = -1;
|
|
|
716
|
|
|
717 /* Remove the file descriptor from the epoll.
|
|
|
718 * This avoids a problem where the same file description remains open
|
|
|
719 * in another process, causing repeated junk epoll events.
|
|
|
720 *
|
|
|
721 * Perform EPOLL_CTL_DEL immediately instead of going through
|
|
|
722 * io_uring's submit queue, otherwise the file descriptor may
|
|
|
723 * be closed by the time the kernel starts the operation.
|
|
|
724 *
|
|
|
725 * We pass in a dummy epoll_event, to work around a bug in old kernels.
|
|
|
726 *
|
|
|
727 * Work around a bug in kernels 3.10 to 3.19 where passing a struct that
|
|
|
728 * has the EPOLLWAKEUP flag set generates spurious audit syslog warnings.
|
|
|
729 */
|
|
|
730 memset(&dummy, 0, sizeof(dummy));
|
|
|
731 epoll_ctl(loop->backend_fd, EPOLL_CTL_DEL, fd, &dummy);
|
|
|
732 }
|
|
|
733
|
|
|
734
|
|
|
735 int uv__io_check_fd(uv_loop_t* loop, int fd) {
|
|
|
736 struct epoll_event e;
|
|
|
737 int rc;
|
|
|
738
|
|
|
739 memset(&e, 0, sizeof(e));
|
|
|
740 e.events = POLLIN;
|
|
|
741 e.data.fd = -1;
|
|
|
742
|
|
|
743 rc = 0;
|
|
|
744 if (epoll_ctl(loop->backend_fd, EPOLL_CTL_ADD, fd, &e))
|
|
|
745 if (errno != EEXIST)
|
|
|
746 rc = UV__ERR(errno);
|
|
|
747
|
|
|
748 if (rc == 0)
|
|
|
749 if (epoll_ctl(loop->backend_fd, EPOLL_CTL_DEL, fd, &e))
|
|
|
750 abort();
|
|
|
751
|
|
|
752 return rc;
|
|
|
753 }
|
|
|
754
|
|
|
755
|
|
|
756 /* Caller must initialize SQE and call uv__iou_submit(). */
|
|
|
757 static struct uv__io_uring_sqe* uv__iou_get_sqe(struct uv__iou* iou,
|
|
|
758 uv_loop_t* loop,
|
|
|
759 uv_fs_t* req) {
|
|
|
760 struct uv__io_uring_sqe* sqe;
|
|
|
761 uint32_t head;
|
|
|
762 uint32_t tail;
|
|
|
763 uint32_t mask;
|
|
|
764 uint32_t slot;
|
|
|
765
|
|
|
766 /* Lazily create the ring. State machine: -2 means uninitialized, -1 means
|
|
|
767 * initialization failed. Anything else is a valid ring file descriptor.
|
|
|
768 */
|
|
|
769 if (iou->ringfd == -2) {
|
|
|
770 /* By default, the SQPOLL is not created. Enable only if the loop is
|
|
|
771 * configured with UV_LOOP_USE_IO_URING_SQPOLL and the UV_USE_IO_URING
|
|
|
772 * environment variable is unset or a positive number.
|
|
|
773 */
|
|
|
774 if (loop->flags & UV_LOOP_ENABLE_IO_URING_SQPOLL)
|
|
|
775 if (uv__use_io_uring(UV__IORING_SETUP_SQPOLL))
|
|
|
776 uv__iou_init(loop->backend_fd, iou, 64, UV__IORING_SETUP_SQPOLL);
|
|
|
777
|
|
|
778 if (iou->ringfd == -2)
|
|
|
779 iou->ringfd = -1; /* "failed" */
|
|
|
780 }
|
|
|
781
|
|
|
782 if (iou->ringfd == -1)
|
|
|
783 return NULL;
|
|
|
784
|
|
|
785 head = atomic_load_explicit((_Atomic uint32_t*) iou->sqhead,
|
|
|
786 memory_order_acquire);
|
|
|
787 tail = *iou->sqtail;
|
|
|
788 mask = iou->sqmask;
|
|
|
789
|
|
|
790 if ((head & mask) == ((tail + 1) & mask))
|
|
|
791 return NULL; /* No room in ring buffer. TODO(bnoordhuis) maybe flush it? */
|
|
|
792
|
|
|
793 slot = tail & mask;
|
|
|
794 sqe = iou->sqe;
|
|
|
795 sqe = &sqe[slot];
|
|
|
796 memset(sqe, 0, sizeof(*sqe));
|
|
|
797 sqe->user_data = (uintptr_t) req;
|
|
|
798
|
|
|
799 /* Pacify uv_cancel(). */
|
|
|
800 req->work_req.loop = loop;
|
|
|
801 req->work_req.work = NULL;
|
|
|
802 req->work_req.done = NULL;
|
|
|
803 uv__queue_init(&req->work_req.wq);
|
|
|
804
|
|
|
805 uv__req_register(loop);
|
|
|
806 iou->in_flight++;
|
|
|
807
|
|
|
808 return sqe;
|
|
|
809 }
|
|
|
810
|
|
|
811
|
|
|
812 static void uv__iou_submit(struct uv__iou* iou) {
|
|
|
813 uint32_t flags;
|
|
|
814
|
|
|
815 atomic_store_explicit((_Atomic uint32_t*) iou->sqtail,
|
|
|
816 *iou->sqtail + 1,
|
|
|
817 memory_order_release);
|
|
|
818
|
|
|
819 flags = atomic_load_explicit((_Atomic uint32_t*) iou->sqflags,
|
|
|
820 memory_order_acquire);
|
|
|
821
|
|
|
822 if (flags & UV__IORING_SQ_NEED_WAKEUP)
|
|
|
823 if (uv__io_uring_enter(iou->ringfd, 0, 0, UV__IORING_ENTER_SQ_WAKEUP))
|
|
|
824 if (errno != EOWNERDEAD) /* Kernel bug. Harmless, ignore. */
|
|
|
825 perror("libuv: io_uring_enter(wakeup)"); /* Can't happen. */
|
|
|
826 }
|
|
|
827
|
|
|
828
|
|
|
829 int uv__iou_fs_close(uv_loop_t* loop, uv_fs_t* req) {
|
|
|
830 struct uv__io_uring_sqe* sqe;
|
|
|
831 struct uv__iou* iou;
|
|
|
832 int kv;
|
|
|
833
|
|
|
834 kv = uv__kernel_version();
|
|
|
835 /* Work around a poorly understood bug in older kernels where closing a file
|
|
|
836 * descriptor pointing to /foo/bar results in ETXTBSY errors when trying to
|
|
|
837 * execve("/foo/bar") later on. The bug seems to have been fixed somewhere
|
|
|
838 * between 5.15.85 and 5.15.90. I couldn't pinpoint the responsible commit
|
|
|
839 * but good candidates are the several data race fixes. Interestingly, it
|
|
|
840 * seems to manifest only when running under Docker so the possibility of
|
|
|
841 * a Docker bug can't be completely ruled out either. Yay, computers.
|
|
|
842 * Also, disable on non-longterm versions between 5.16.0 (non-longterm) and
|
|
|
843 * 6.1.0 (longterm). Starting with longterm 6.1.x, the issue seems to be
|
|
|
844 * solved.
|
|
|
845 */
|
|
|
846 if (kv < /* 5.15.90 */ 0x050F5A)
|
|
|
847 return 0;
|
|
|
848
|
|
|
849 if (kv >= /* 5.16.0 */ 0x050A00 && kv < /* 6.1.0 */ 0x060100)
|
|
|
850 return 0;
|
|
|
851
|
|
|
852
|
|
|
853 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
854
|
|
|
855 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
856 if (sqe == NULL)
|
|
|
857 return 0;
|
|
|
858
|
|
|
859 sqe->fd = req->file;
|
|
|
860 sqe->opcode = UV__IORING_OP_CLOSE;
|
|
|
861
|
|
|
862 uv__iou_submit(iou);
|
|
|
863
|
|
|
864 return 1;
|
|
|
865 }
|
|
|
866
|
|
|
867
|
|
|
868 int uv__iou_fs_ftruncate(uv_loop_t* loop, uv_fs_t* req) {
|
|
|
869 struct uv__io_uring_sqe* sqe;
|
|
|
870 struct uv__iou* iou;
|
|
|
871
|
|
|
872 if (uv__kernel_version() < /* 6.9 */0x060900)
|
|
|
873 return 0;
|
|
|
874
|
|
|
875 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
876 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
877 if (sqe == NULL)
|
|
|
878 return 0;
|
|
|
879
|
|
|
880 sqe->fd = req->file;
|
|
|
881 sqe->len = req->off;
|
|
|
882 sqe->opcode = UV__IORING_OP_FTRUNCATE;
|
|
|
883 uv__iou_submit(iou);
|
|
|
884
|
|
|
885 return 1;
|
|
|
886 }
|
|
|
887
|
|
|
888 int uv__iou_fs_fsync_or_fdatasync(uv_loop_t* loop,
|
|
|
889 uv_fs_t* req,
|
|
|
890 uint32_t fsync_flags) {
|
|
|
891 struct uv__io_uring_sqe* sqe;
|
|
|
892 struct uv__iou* iou;
|
|
|
893
|
|
|
894 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
895
|
|
|
896 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
897 if (sqe == NULL)
|
|
|
898 return 0;
|
|
|
899
|
|
|
900 /* Little known fact: setting seq->off and seq->len turns
|
|
|
901 * it into an asynchronous sync_file_range() operation.
|
|
|
902 */
|
|
|
903 sqe->fd = req->file;
|
|
|
904 sqe->fsync_flags = fsync_flags;
|
|
|
905 sqe->opcode = UV__IORING_OP_FSYNC;
|
|
|
906
|
|
|
907 uv__iou_submit(iou);
|
|
|
908
|
|
|
909 return 1;
|
|
|
910 }
|
|
|
911
|
|
|
912
|
|
|
913 int uv__iou_fs_link(uv_loop_t* loop, uv_fs_t* req) {
|
|
|
914 struct uv__io_uring_sqe* sqe;
|
|
|
915 struct uv__iou* iou;
|
|
|
916
|
|
|
917 if (uv__kernel_version() < /* 5.15.0 */0x050F00)
|
|
|
918 return 0;
|
|
|
919
|
|
|
920 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
921 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
922 if (sqe == NULL)
|
|
|
923 return 0;
|
|
|
924
|
|
|
925 sqe->addr = (uintptr_t) req->path;
|
|
|
926 sqe->fd = AT_FDCWD;
|
|
|
927 sqe->addr2 = (uintptr_t) req->new_path;
|
|
|
928 sqe->len = AT_FDCWD;
|
|
|
929 sqe->opcode = UV__IORING_OP_LINKAT;
|
|
|
930
|
|
|
931 uv__iou_submit(iou);
|
|
|
932
|
|
|
933 return 1;
|
|
|
934 }
|
|
|
935
|
|
|
936
|
|
|
937 int uv__iou_fs_mkdir(uv_loop_t* loop, uv_fs_t* req) {
|
|
|
938 struct uv__io_uring_sqe* sqe;
|
|
|
939 struct uv__iou* iou;
|
|
|
940
|
|
|
941 if (uv__kernel_version() < /* 5.15.0 */0x050F00)
|
|
|
942 return 0;
|
|
|
943
|
|
|
944 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
945 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
946 if (sqe == NULL)
|
|
|
947 return 0;
|
|
|
948
|
|
|
949 sqe->addr = (uintptr_t) req->path;
|
|
|
950 sqe->fd = AT_FDCWD;
|
|
|
951 sqe->len = req->mode;
|
|
|
952 sqe->opcode = UV__IORING_OP_MKDIRAT;
|
|
|
953
|
|
|
954 uv__iou_submit(iou);
|
|
|
955
|
|
|
956 return 1;
|
|
|
957 }
|
|
|
958
|
|
|
959
|
|
|
960 int uv__iou_fs_open(uv_loop_t* loop, uv_fs_t* req) {
|
|
|
961 struct uv__io_uring_sqe* sqe;
|
|
|
962 struct uv__iou* iou;
|
|
|
963
|
|
|
964 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
965
|
|
|
966 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
967 if (sqe == NULL)
|
|
|
968 return 0;
|
|
|
969
|
|
|
970 sqe->addr = (uintptr_t) req->path;
|
|
|
971 sqe->fd = AT_FDCWD;
|
|
|
972 sqe->len = req->mode;
|
|
|
973 sqe->opcode = UV__IORING_OP_OPENAT;
|
|
|
974 sqe->open_flags = req->flags | O_CLOEXEC;
|
|
|
975
|
|
|
976 uv__iou_submit(iou);
|
|
|
977
|
|
|
978 return 1;
|
|
|
979 }
|
|
|
980
|
|
|
981
|
|
|
982 int uv__iou_fs_rename(uv_loop_t* loop, uv_fs_t* req) {
|
|
|
983 struct uv__io_uring_sqe* sqe;
|
|
|
984 struct uv__iou* iou;
|
|
|
985
|
|
|
986 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
987
|
|
|
988 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
989 if (sqe == NULL)
|
|
|
990 return 0;
|
|
|
991
|
|
|
992 sqe->addr = (uintptr_t) req->path;
|
|
|
993 sqe->fd = AT_FDCWD;
|
|
|
994 sqe->addr2 = (uintptr_t) req->new_path;
|
|
|
995 sqe->len = AT_FDCWD;
|
|
|
996 sqe->opcode = UV__IORING_OP_RENAMEAT;
|
|
|
997
|
|
|
998 uv__iou_submit(iou);
|
|
|
999
|
|
|
1000 return 1;
|
|
|
1001 }
|
|
|
1002
|
|
|
1003
|
|
|
1004 int uv__iou_fs_symlink(uv_loop_t* loop, uv_fs_t* req) {
|
|
|
1005 struct uv__io_uring_sqe* sqe;
|
|
|
1006 struct uv__iou* iou;
|
|
|
1007
|
|
|
1008 if (uv__kernel_version() < /* 5.15.0 */0x050F00)
|
|
|
1009 return 0;
|
|
|
1010
|
|
|
1011 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
1012 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
1013 if (sqe == NULL)
|
|
|
1014 return 0;
|
|
|
1015
|
|
|
1016 sqe->addr = (uintptr_t) req->path;
|
|
|
1017 sqe->fd = AT_FDCWD;
|
|
|
1018 sqe->addr2 = (uintptr_t) req->new_path;
|
|
|
1019 sqe->opcode = UV__IORING_OP_SYMLINKAT;
|
|
|
1020
|
|
|
1021 uv__iou_submit(iou);
|
|
|
1022
|
|
|
1023 return 1;
|
|
|
1024 }
|
|
|
1025
|
|
|
1026
|
|
|
1027 int uv__iou_fs_unlink(uv_loop_t* loop, uv_fs_t* req) {
|
|
|
1028 struct uv__io_uring_sqe* sqe;
|
|
|
1029 struct uv__iou* iou;
|
|
|
1030
|
|
|
1031 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
1032
|
|
|
1033 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
1034 if (sqe == NULL)
|
|
|
1035 return 0;
|
|
|
1036
|
|
|
1037 sqe->addr = (uintptr_t) req->path;
|
|
|
1038 sqe->fd = AT_FDCWD;
|
|
|
1039 sqe->opcode = UV__IORING_OP_UNLINKAT;
|
|
|
1040
|
|
|
1041 uv__iou_submit(iou);
|
|
|
1042
|
|
|
1043 return 1;
|
|
|
1044 }
|
|
|
1045
|
|
|
1046
|
|
|
1047 int uv__iou_fs_read_or_write(uv_loop_t* loop,
|
|
|
1048 uv_fs_t* req,
|
|
|
1049 int is_read) {
|
|
|
1050 struct uv__io_uring_sqe* sqe;
|
|
|
1051 struct uv__iou* iou;
|
|
|
1052
|
|
|
1053 /* If iovcnt is greater than IOV_MAX, cap it to IOV_MAX on reads and fallback
|
|
|
1054 * to the threadpool on writes */
|
|
|
1055 if (req->nbufs > IOV_MAX) {
|
|
|
1056 if (is_read)
|
|
|
1057 req->nbufs = IOV_MAX;
|
|
|
1058 else
|
|
|
1059 return 0;
|
|
|
1060 }
|
|
|
1061
|
|
|
1062 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
1063
|
|
|
1064 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
1065 if (sqe == NULL)
|
|
|
1066 return 0;
|
|
|
1067
|
|
|
1068 sqe->addr = (uintptr_t) req->bufs;
|
|
|
1069 sqe->fd = req->file;
|
|
|
1070 sqe->len = req->nbufs;
|
|
|
1071 sqe->off = req->off < 0 ? -1 : req->off;
|
|
|
1072 sqe->opcode = is_read ? UV__IORING_OP_READV : UV__IORING_OP_WRITEV;
|
|
|
1073
|
|
|
1074 uv__iou_submit(iou);
|
|
|
1075
|
|
|
1076 return 1;
|
|
|
1077 }
|
|
|
1078
|
|
|
1079
|
|
|
1080 int uv__iou_fs_statx(uv_loop_t* loop,
|
|
|
1081 uv_fs_t* req,
|
|
|
1082 int is_fstat,
|
|
|
1083 int is_lstat) {
|
|
|
1084 struct uv__io_uring_sqe* sqe;
|
|
|
1085 struct uv__statx* statxbuf;
|
|
|
1086 struct uv__iou* iou;
|
|
|
1087
|
|
|
1088 statxbuf = uv__malloc(sizeof(*statxbuf));
|
|
|
1089 if (statxbuf == NULL)
|
|
|
1090 return 0;
|
|
|
1091
|
|
|
1092 iou = &uv__get_internal_fields(loop)->iou;
|
|
|
1093
|
|
|
1094 sqe = uv__iou_get_sqe(iou, loop, req);
|
|
|
1095 if (sqe == NULL) {
|
|
|
1096 uv__free(statxbuf);
|
|
|
1097 return 0;
|
|
|
1098 }
|
|
|
1099
|
|
|
1100 req->ptr = statxbuf;
|
|
|
1101
|
|
|
1102 sqe->addr = (uintptr_t) req->path;
|
|
|
1103 sqe->addr2 = (uintptr_t) statxbuf;
|
|
|
1104 sqe->fd = AT_FDCWD;
|
|
|
1105 sqe->len = 0xFFF; /* STATX_BASIC_STATS + STATX_BTIME */
|
|
|
1106 sqe->opcode = UV__IORING_OP_STATX;
|
|
|
1107
|
|
|
1108 if (is_fstat) {
|
|
|
1109 sqe->addr = (uintptr_t) "";
|
|
|
1110 sqe->fd = req->file;
|
|
|
1111 sqe->statx_flags |= 0x1000; /* AT_EMPTY_PATH */
|
|
|
1112 }
|
|
|
1113
|
|
|
1114 if (is_lstat)
|
|
|
1115 sqe->statx_flags |= AT_SYMLINK_NOFOLLOW;
|
|
|
1116
|
|
|
1117 uv__iou_submit(iou);
|
|
|
1118
|
|
|
1119 return 1;
|
|
|
1120 }
|
|
|
1121
|
|
|
1122
|
|
|
1123 void uv__statx_to_stat(const struct uv__statx* statxbuf, uv_stat_t* buf) {
|
|
|
1124 buf->st_dev = makedev(statxbuf->stx_dev_major, statxbuf->stx_dev_minor);
|
|
|
1125 buf->st_mode = statxbuf->stx_mode;
|
|
|
1126 buf->st_nlink = statxbuf->stx_nlink;
|
|
|
1127 buf->st_uid = statxbuf->stx_uid;
|
|
|
1128 buf->st_gid = statxbuf->stx_gid;
|
|
|
1129 buf->st_rdev = makedev(statxbuf->stx_rdev_major, statxbuf->stx_rdev_minor);
|
|
|
1130 buf->st_ino = statxbuf->stx_ino;
|
|
|
1131 buf->st_size = statxbuf->stx_size;
|
|
|
1132 buf->st_blksize = statxbuf->stx_blksize;
|
|
|
1133 buf->st_blocks = statxbuf->stx_blocks;
|
|
|
1134 buf->st_atim.tv_sec = statxbuf->stx_atime.tv_sec;
|
|
|
1135 buf->st_atim.tv_nsec = statxbuf->stx_atime.tv_nsec;
|
|
|
1136 buf->st_mtim.tv_sec = statxbuf->stx_mtime.tv_sec;
|
|
|
1137 buf->st_mtim.tv_nsec = statxbuf->stx_mtime.tv_nsec;
|
|
|
1138 buf->st_ctim.tv_sec = statxbuf->stx_ctime.tv_sec;
|
|
|
1139 buf->st_ctim.tv_nsec = statxbuf->stx_ctime.tv_nsec;
|
|
|
1140 buf->st_birthtim.tv_sec = statxbuf->stx_btime.tv_sec;
|
|
|
1141 buf->st_birthtim.tv_nsec = statxbuf->stx_btime.tv_nsec;
|
|
|
1142 buf->st_flags = 0;
|
|
|
1143 buf->st_gen = 0;
|
|
|
1144 }
|
|
|
1145
|
|
|
1146
|
|
|
1147 static void uv__iou_fs_statx_post(uv_fs_t* req) {
|
|
|
1148 struct uv__statx* statxbuf;
|
|
|
1149 uv_stat_t* buf;
|
|
|
1150
|
|
|
1151 buf = &req->statbuf;
|
|
|
1152 statxbuf = req->ptr;
|
|
|
1153 req->ptr = NULL;
|
|
|
1154
|
|
|
1155 if (req->result == 0) {
|
|
|
1156 uv__msan_unpoison(statxbuf, sizeof(*statxbuf));
|
|
|
1157 uv__statx_to_stat(statxbuf, buf);
|
|
|
1158 req->ptr = buf;
|
|
|
1159 }
|
|
|
1160
|
|
|
1161 uv__free(statxbuf);
|
|
|
1162 }
|
|
|
1163
|
|
|
1164
|
|
|
1165 static void uv__poll_io_uring(uv_loop_t* loop, struct uv__iou* iou) {
|
|
|
1166 struct uv__io_uring_cqe* cqe;
|
|
|
1167 struct uv__io_uring_cqe* e;
|
|
|
1168 uv_fs_t* req;
|
|
|
1169 uint32_t head;
|
|
|
1170 uint32_t tail;
|
|
|
1171 uint32_t mask;
|
|
|
1172 uint32_t i;
|
|
|
1173 uint32_t flags;
|
|
|
1174 int nevents;
|
|
|
1175 int rc;
|
|
|
1176
|
|
|
1177 head = *iou->cqhead;
|
|
|
1178 tail = atomic_load_explicit((_Atomic uint32_t*) iou->cqtail,
|
|
|
1179 memory_order_acquire);
|
|
|
1180 mask = iou->cqmask;
|
|
|
1181 cqe = iou->cqe;
|
|
|
1182 nevents = 0;
|
|
|
1183
|
|
|
1184 for (i = head; i != tail; i++) {
|
|
|
1185 e = &cqe[i & mask];
|
|
|
1186
|
|
|
1187 req = (uv_fs_t*) (uintptr_t) e->user_data;
|
|
|
1188 assert(req->type == UV_FS);
|
|
|
1189
|
|
|
1190 uv__req_unregister(loop);
|
|
|
1191 iou->in_flight--;
|
|
|
1192
|
|
|
1193 /* If the op is not supported by the kernel retry using the thread pool */
|
|
|
1194 if (e->res == -EOPNOTSUPP) {
|
|
|
1195 uv__fs_post(loop, req);
|
|
|
1196 continue;
|
|
|
1197 }
|
|
|
1198
|
|
|
1199 /* io_uring stores error codes as negative numbers, same as libuv. */
|
|
|
1200 req->result = e->res;
|
|
|
1201
|
|
|
1202 switch (req->fs_type) {
|
|
|
1203 case UV_FS_FSTAT:
|
|
|
1204 case UV_FS_LSTAT:
|
|
|
1205 case UV_FS_STAT:
|
|
|
1206 uv__iou_fs_statx_post(req);
|
|
|
1207 break;
|
|
|
1208 default: /* Squelch -Wswitch warnings. */
|
|
|
1209 break;
|
|
|
1210 }
|
|
|
1211
|
|
|
1212 uv__metrics_update_idle_time(loop);
|
|
|
1213 req->cb(req);
|
|
|
1214 nevents++;
|
|
|
1215 }
|
|
|
1216
|
|
|
1217 atomic_store_explicit((_Atomic uint32_t*) iou->cqhead,
|
|
|
1218 tail,
|
|
|
1219 memory_order_release);
|
|
|
1220
|
|
|
1221 /* Check whether CQE's overflowed, if so enter the kernel to make them
|
|
|
1222 * available. Don't grab them immediately but in the next loop iteration to
|
|
|
1223 * avoid loop starvation. */
|
|
|
1224 flags = atomic_load_explicit((_Atomic uint32_t*) iou->sqflags,
|
|
|
1225 memory_order_acquire);
|
|
|
1226
|
|
|
1227 if (flags & UV__IORING_SQ_CQ_OVERFLOW) {
|
|
|
1228 do
|
|
|
1229 rc = uv__io_uring_enter(iou->ringfd, 0, 0, UV__IORING_ENTER_GETEVENTS);
|
|
|
1230 while (rc == -1 && errno == EINTR);
|
|
|
1231
|
|
|
1232 if (rc < 0)
|
|
|
1233 perror("libuv: io_uring_enter(getevents)"); /* Can't happen. */
|
|
|
1234 }
|
|
|
1235
|
|
|
1236 uv__metrics_inc_events(loop, nevents);
|
|
|
1237 if (uv__get_internal_fields(loop)->current_timeout == 0)
|
|
|
1238 uv__metrics_inc_events_waiting(loop, nevents);
|
|
|
1239 }
|
|
|
1240
|
|
|
1241
|
|
|
1242 /* Only for EPOLL_CTL_ADD and EPOLL_CTL_MOD. EPOLL_CTL_DEL should always be
|
|
|
1243 * executed immediately, otherwise the file descriptor may have been closed
|
|
|
1244 * by the time the kernel starts the operation.
|
|
|
1245 */
|
|
|
1246 static void uv__epoll_ctl_prep(int epollfd,
|
|
|
1247 struct uv__iou* ctl,
|
|
|
1248 struct epoll_event (*events)[256],
|
|
|
1249 int op,
|
|
|
1250 int fd,
|
|
|
1251 struct epoll_event* e) {
|
|
|
1252 struct uv__io_uring_sqe* sqe;
|
|
|
1253 struct epoll_event* pe;
|
|
|
1254 uint32_t mask;
|
|
|
1255 uint32_t slot;
|
|
|
1256
|
|
|
1257 assert(op == EPOLL_CTL_ADD || op == EPOLL_CTL_MOD);
|
|
|
1258 assert(ctl->ringfd != -1);
|
|
|
1259
|
|
|
1260 mask = ctl->sqmask;
|
|
|
1261 slot = (*ctl->sqtail)++ & mask;
|
|
|
1262
|
|
|
1263 pe = &(*events)[slot];
|
|
|
1264 *pe = *e;
|
|
|
1265
|
|
|
1266 sqe = ctl->sqe;
|
|
|
1267 sqe = &sqe[slot];
|
|
|
1268
|
|
|
1269 memset(sqe, 0, sizeof(*sqe));
|
|
|
1270 sqe->addr = (uintptr_t) pe;
|
|
|
1271 sqe->fd = epollfd;
|
|
|
1272 sqe->len = op;
|
|
|
1273 sqe->off = fd;
|
|
|
1274 sqe->opcode = UV__IORING_OP_EPOLL_CTL;
|
|
|
1275 sqe->user_data = op | slot << 2 | (int64_t) fd << 32;
|
|
|
1276
|
|
|
1277 if ((*ctl->sqhead & mask) == (*ctl->sqtail & mask))
|
|
|
1278 uv__epoll_ctl_flush(epollfd, ctl, events);
|
|
|
1279 }
|
|
|
1280
|
|
|
1281
|
|
|
1282 static void uv__epoll_ctl_flush(int epollfd,
|
|
|
1283 struct uv__iou* ctl,
|
|
|
1284 struct epoll_event (*events)[256]) {
|
|
|
1285 struct epoll_event oldevents[256];
|
|
|
1286 struct uv__io_uring_cqe* cqe;
|
|
|
1287 uint32_t oldslot;
|
|
|
1288 uint32_t slot;
|
|
|
1289 uint32_t n;
|
|
|
1290 int fd;
|
|
|
1291 int op;
|
|
|
1292 int rc;
|
|
|
1293
|
|
|
1294 STATIC_ASSERT(sizeof(oldevents) == sizeof(*events));
|
|
|
1295 assert(ctl->ringfd != -1);
|
|
|
1296 assert(*ctl->sqhead != *ctl->sqtail);
|
|
|
1297
|
|
|
1298 n = *ctl->sqtail - *ctl->sqhead;
|
|
|
1299 do
|
|
|
1300 rc = uv__io_uring_enter(ctl->ringfd, n, n, UV__IORING_ENTER_GETEVENTS);
|
|
|
1301 while (rc == -1 && errno == EINTR);
|
|
|
1302
|
|
|
1303 if (rc < 0)
|
|
|
1304 perror("libuv: io_uring_enter(getevents)"); /* Can't happen. */
|
|
|
1305
|
|
|
1306 if (rc != (int) n)
|
|
|
1307 abort();
|
|
|
1308
|
|
|
1309 assert(*ctl->sqhead == *ctl->sqtail);
|
|
|
1310
|
|
|
1311 memcpy(oldevents, *events, sizeof(*events));
|
|
|
1312
|
|
|
1313 /* Failed submissions are either EPOLL_CTL_DEL commands for file descriptors
|
|
|
1314 * that have been closed, or EPOLL_CTL_ADD commands for file descriptors
|
|
|
1315 * that we are already watching. Ignore the former and retry the latter
|
|
|
1316 * with EPOLL_CTL_MOD.
|
|
|
1317 */
|
|
|
1318 while (*ctl->cqhead != *ctl->cqtail) {
|
|
|
1319 slot = (*ctl->cqhead)++ & ctl->cqmask;
|
|
|
1320
|
|
|
1321 cqe = ctl->cqe;
|
|
|
1322 cqe = &cqe[slot];
|
|
|
1323
|
|
|
1324 if (cqe->res == 0)
|
|
|
1325 continue;
|
|
|
1326
|
|
|
1327 fd = cqe->user_data >> 32;
|
|
|
1328 op = 3 & cqe->user_data;
|
|
|
1329 oldslot = 255 & (cqe->user_data >> 2);
|
|
|
1330
|
|
|
1331 if (op == EPOLL_CTL_DEL)
|
|
|
1332 continue;
|
|
|
1333
|
|
|
1334 if (op != EPOLL_CTL_ADD)
|
|
|
1335 abort();
|
|
|
1336
|
|
|
1337 if (cqe->res != -EEXIST)
|
|
|
1338 abort();
|
|
|
1339
|
|
|
1340 uv__epoll_ctl_prep(epollfd,
|
|
|
1341 ctl,
|
|
|
1342 events,
|
|
|
1343 EPOLL_CTL_MOD,
|
|
|
1344 fd,
|
|
|
1345 &oldevents[oldslot]);
|
|
|
1346 }
|
|
|
1347 }
|
|
|
1348
|
|
|
1349
|
|
|
1350 void uv__io_poll(uv_loop_t* loop, int timeout) {
|
|
|
1351 uv__loop_internal_fields_t* lfields;
|
|
|
1352 struct epoll_event events[1024];
|
|
|
1353 struct epoll_event prep[256];
|
|
|
1354 struct uv__invalidate inv;
|
|
|
1355 struct epoll_event* pe;
|
|
|
1356 struct epoll_event e;
|
|
|
1357 struct uv__iou* ctl;
|
|
|
1358 struct uv__iou* iou;
|
|
|
1359 int real_timeout;
|
|
|
1360 struct uv__queue* q;
|
|
|
1361 uv__io_t* w;
|
|
|
1362 sigset_t* sigmask;
|
|
|
1363 sigset_t sigset;
|
|
|
1364 uint64_t base;
|
|
|
1365 int have_iou_events;
|
|
|
1366 int have_signals;
|
|
|
1367 int nevents;
|
|
|
1368 int epollfd;
|
|
|
1369 int count;
|
|
|
1370 int nfds;
|
|
|
1371 int fd;
|
|
|
1372 int op;
|
|
|
1373 int i;
|
|
|
1374 int user_timeout;
|
|
|
1375 int reset_timeout;
|
|
|
1376
|
|
|
1377 lfields = uv__get_internal_fields(loop);
|
|
|
1378 ctl = &lfields->ctl;
|
|
|
1379 iou = &lfields->iou;
|
|
|
1380
|
|
|
1381 sigmask = NULL;
|
|
|
1382 if (loop->flags & UV_LOOP_BLOCK_SIGPROF) {
|
|
|
1383 sigemptyset(&sigset);
|
|
|
1384 sigaddset(&sigset, SIGPROF);
|
|
|
1385 sigmask = &sigset;
|
|
|
1386 }
|
|
|
1387
|
|
|
1388 assert(timeout >= -1);
|
|
|
1389 base = loop->time;
|
|
|
1390 count = 48; /* Benchmarks suggest this gives the best throughput. */
|
|
|
1391 real_timeout = timeout;
|
|
|
1392
|
|
|
1393 if (lfields->flags & UV_METRICS_IDLE_TIME) {
|
|
|
1394 reset_timeout = 1;
|
|
|
1395 user_timeout = timeout;
|
|
|
1396 timeout = 0;
|
|
|
1397 } else {
|
|
|
1398 reset_timeout = 0;
|
|
|
1399 user_timeout = 0;
|
|
|
1400 }
|
|
|
1401
|
|
|
1402 epollfd = loop->backend_fd;
|
|
|
1403
|
|
|
1404 memset(&e, 0, sizeof(e));
|
|
|
1405
|
|
|
1406 while (!uv__queue_empty(&loop->watcher_queue)) {
|
|
|
1407 q = uv__queue_head(&loop->watcher_queue);
|
|
|
1408 w = uv__queue_data(q, uv__io_t, watcher_queue);
|
|
|
1409 uv__queue_remove(q);
|
|
|
1410 uv__queue_init(q);
|
|
|
1411
|
|
|
1412 op = EPOLL_CTL_MOD;
|
|
|
1413 if (w->events == 0)
|
|
|
1414 op = EPOLL_CTL_ADD;
|
|
|
1415
|
|
|
1416 w->events = w->pevents;
|
|
|
1417 e.events = w->pevents;
|
|
|
1418 e.data.fd = w->fd;
|
|
|
1419 fd = w->fd;
|
|
|
1420
|
|
|
1421 if (ctl->ringfd != -1) {
|
|
|
1422 uv__epoll_ctl_prep(epollfd, ctl, &prep, op, fd, &e);
|
|
|
1423 continue;
|
|
|
1424 }
|
|
|
1425
|
|
|
1426 if (!epoll_ctl(epollfd, op, fd, &e))
|
|
|
1427 continue;
|
|
|
1428
|
|
|
1429 assert(op == EPOLL_CTL_ADD);
|
|
|
1430 assert(errno == EEXIST);
|
|
|
1431
|
|
|
1432 /* File descriptor that's been watched before, update event mask. */
|
|
|
1433 if (epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &e))
|
|
|
1434 abort();
|
|
|
1435 }
|
|
|
1436
|
|
|
1437 inv.events = events;
|
|
|
1438 inv.prep = &prep;
|
|
|
1439 inv.nfds = -1;
|
|
|
1440
|
|
|
1441 for (;;) {
|
|
|
1442 if (loop->nfds == 0)
|
|
|
1443 if (iou->in_flight == 0)
|
|
|
1444 break;
|
|
|
1445
|
|
|
1446 /* All event mask mutations should be visible to the kernel before
|
|
|
1447 * we enter epoll_pwait().
|
|
|
1448 */
|
|
|
1449 if (ctl->ringfd != -1)
|
|
|
1450 while (*ctl->sqhead != *ctl->sqtail)
|
|
|
1451 uv__epoll_ctl_flush(epollfd, ctl, &prep);
|
|
|
1452
|
|
|
1453 /* Only need to set the provider_entry_time if timeout != 0. The function
|
|
|
1454 * will return early if the loop isn't configured with UV_METRICS_IDLE_TIME.
|
|
|
1455 */
|
|
|
1456 if (timeout != 0)
|
|
|
1457 uv__metrics_set_provider_entry_time(loop);
|
|
|
1458
|
|
|
1459 /* Store the current timeout in a location that's globally accessible so
|
|
|
1460 * other locations like uv__work_done() can determine whether the queue
|
|
|
1461 * of events in the callback were waiting when poll was called.
|
|
|
1462 */
|
|
|
1463 lfields->current_timeout = timeout;
|
|
|
1464
|
|
|
1465 nfds = epoll_pwait(epollfd, events, ARRAY_SIZE(events), timeout, sigmask);
|
|
|
1466
|
|
|
1467 /* Update loop->time unconditionally. It's tempting to skip the update when
|
|
|
1468 * timeout == 0 (i.e. non-blocking poll) but there is no guarantee that the
|
|
|
1469 * operating system didn't reschedule our process while in the syscall.
|
|
|
1470 */
|
|
|
1471 SAVE_ERRNO(uv__update_time(loop));
|
|
|
1472
|
|
|
1473 if (nfds == -1)
|
|
|
1474 assert(errno == EINTR);
|
|
|
1475 else if (nfds == 0)
|
|
|
1476 /* Unlimited timeout should only return with events or signal. */
|
|
|
1477 assert(timeout != -1);
|
|
|
1478
|
|
|
1479 if (nfds == 0 || nfds == -1) {
|
|
|
1480 if (reset_timeout != 0) {
|
|
|
1481 timeout = user_timeout;
|
|
|
1482 reset_timeout = 0;
|
|
|
1483 } else if (nfds == 0) {
|
|
|
1484 return;
|
|
|
1485 }
|
|
|
1486
|
|
|
1487 /* Interrupted by a signal. Update timeout and poll again. */
|
|
|
1488 goto update_timeout;
|
|
|
1489 }
|
|
|
1490
|
|
|
1491 have_iou_events = 0;
|
|
|
1492 have_signals = 0;
|
|
|
1493 nevents = 0;
|
|
|
1494
|
|
|
1495 inv.nfds = nfds;
|
|
|
1496 lfields->inv = &inv;
|
|
|
1497
|
|
|
1498 for (i = 0; i < nfds; i++) {
|
|
|
1499 pe = events + i;
|
|
|
1500 fd = pe->data.fd;
|
|
|
1501
|
|
|
1502 /* Skip invalidated events, see uv__platform_invalidate_fd */
|
|
|
1503 if (fd == -1)
|
|
|
1504 continue;
|
|
|
1505
|
|
|
1506 if (fd == iou->ringfd) {
|
|
|
1507 uv__poll_io_uring(loop, iou);
|
|
|
1508 have_iou_events = 1;
|
|
|
1509 continue;
|
|
|
1510 }
|
|
|
1511
|
|
|
1512 assert(fd >= 0);
|
|
|
1513 assert((unsigned) fd < loop->nwatchers);
|
|
|
1514
|
|
|
1515 w = loop->watchers[fd];
|
|
|
1516
|
|
|
1517 if (w == NULL) {
|
|
|
1518 /* File descriptor that we've stopped watching, disarm it.
|
|
|
1519 *
|
|
|
1520 * Ignore all errors because we may be racing with another thread
|
|
|
1521 * when the file descriptor is closed.
|
|
|
1522 *
|
|
|
1523 * Perform EPOLL_CTL_DEL immediately instead of going through
|
|
|
1524 * io_uring's submit queue, otherwise the file descriptor may
|
|
|
1525 * be closed by the time the kernel starts the operation.
|
|
|
1526 */
|
|
|
1527 epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, pe);
|
|
|
1528 continue;
|
|
|
1529 }
|
|
|
1530
|
|
|
1531 /* Give users only events they're interested in. Prevents spurious
|
|
|
1532 * callbacks when previous callback invocation in this loop has stopped
|
|
|
1533 * the current watcher. Also, filters out events that users has not
|
|
|
1534 * requested us to watch.
|
|
|
1535 */
|
|
|
1536 pe->events &= w->pevents | POLLERR | POLLHUP;
|
|
|
1537
|
|
|
1538 /* Work around an epoll quirk where it sometimes reports just the
|
|
|
1539 * EPOLLERR or EPOLLHUP event. In order to force the event loop to
|
|
|
1540 * move forward, we merge in the read/write events that the watcher
|
|
|
1541 * is interested in; uv__read() and uv__write() will then deal with
|
|
|
1542 * the error or hangup in the usual fashion.
|
|
|
1543 *
|
|
|
1544 * Note to self: happens when epoll reports EPOLLIN|EPOLLHUP, the user
|
|
|
1545 * reads the available data, calls uv_read_stop(), then sometime later
|
|
|
1546 * calls uv_read_start() again. By then, libuv has forgotten about the
|
|
|
1547 * hangup and the kernel won't report EPOLLIN again because there's
|
|
|
1548 * nothing left to read. If anything, libuv is to blame here. The
|
|
|
1549 * current hack is just a quick bandaid; to properly fix it, libuv
|
|
|
1550 * needs to remember the error/hangup event. We should get that for
|
|
|
1551 * free when we switch over to edge-triggered I/O.
|
|
|
1552 */
|
|
|
1553 if (pe->events == POLLERR || pe->events == POLLHUP)
|
|
|
1554 pe->events |=
|
|
|
1555 w->pevents & (POLLIN | POLLOUT | UV__POLLRDHUP | UV__POLLPRI);
|
|
|
1556
|
|
|
1557 if (pe->events != 0) {
|
|
|
1558 /* Run signal watchers last. This also affects child process watchers
|
|
|
1559 * because those are implemented in terms of signal watchers.
|
|
|
1560 */
|
|
|
1561 if (w == &loop->signal_io_watcher) {
|
|
|
1562 have_signals = 1;
|
|
|
1563 } else {
|
|
|
1564 uv__metrics_update_idle_time(loop);
|
|
|
1565 w->cb(loop, w, pe->events);
|
|
|
1566 }
|
|
|
1567
|
|
|
1568 nevents++;
|
|
|
1569 }
|
|
|
1570 }
|
|
|
1571
|
|
|
1572 uv__metrics_inc_events(loop, nevents);
|
|
|
1573 if (reset_timeout != 0) {
|
|
|
1574 timeout = user_timeout;
|
|
|
1575 reset_timeout = 0;
|
|
|
1576 uv__metrics_inc_events_waiting(loop, nevents);
|
|
|
1577 }
|
|
|
1578
|
|
|
1579 if (have_signals != 0) {
|
|
|
1580 uv__metrics_update_idle_time(loop);
|
|
|
1581 loop->signal_io_watcher.cb(loop, &loop->signal_io_watcher, POLLIN);
|
|
|
1582 }
|
|
|
1583
|
|
|
1584 lfields->inv = NULL;
|
|
|
1585
|
|
|
1586 if (have_iou_events != 0)
|
|
|
1587 break; /* Event loop should cycle now so don't poll again. */
|
|
|
1588
|
|
|
1589 if (have_signals != 0)
|
|
|
1590 break; /* Event loop should cycle now so don't poll again. */
|
|
|
1591
|
|
|
1592 if (nevents != 0) {
|
|
|
1593 if (nfds == ARRAY_SIZE(events) && --count != 0) {
|
|
|
1594 /* Poll for more events but don't block this time. */
|
|
|
1595 timeout = 0;
|
|
|
1596 continue;
|
|
|
1597 }
|
|
|
1598 break;
|
|
|
1599 }
|
|
|
1600
|
|
|
1601 update_timeout:
|
|
|
1602 if (timeout == 0)
|
|
|
1603 break;
|
|
|
1604
|
|
|
1605 if (timeout == -1)
|
|
|
1606 continue;
|
|
|
1607
|
|
|
1608 assert(timeout > 0);
|
|
|
1609
|
|
|
1610 real_timeout -= (loop->time - base);
|
|
|
1611 if (real_timeout <= 0)
|
|
|
1612 break;
|
|
|
1613
|
|
|
1614 timeout = real_timeout;
|
|
|
1615 }
|
|
|
1616
|
|
|
1617 if (ctl->ringfd != -1)
|
|
|
1618 while (*ctl->sqhead != *ctl->sqtail)
|
|
|
1619 uv__epoll_ctl_flush(epollfd, ctl, &prep);
|
|
|
1620 }
|
|
|
1621
|
|
|
1622 uint64_t uv__hrtime(uv_clocktype_t type) {
|
|
|
1623 static _Atomic clock_t fast_clock_id = -1;
|
|
|
1624 struct timespec t;
|
|
|
1625 clock_t clock_id;
|
|
|
1626
|
|
|
1627 /* Prefer CLOCK_MONOTONIC_COARSE if available but only when it has
|
|
|
1628 * millisecond granularity or better. CLOCK_MONOTONIC_COARSE is
|
|
|
1629 * serviced entirely from the vDSO, whereas CLOCK_MONOTONIC may
|
|
|
1630 * decide to make a costly system call.
|
|
|
1631 */
|
|
|
1632 /* TODO(bnoordhuis) Use CLOCK_MONOTONIC_COARSE for UV_CLOCK_PRECISE
|
|
|
1633 * when it has microsecond granularity or better (unlikely).
|
|
|
1634 */
|
|
|
1635 clock_id = CLOCK_MONOTONIC;
|
|
|
1636 if (type != UV_CLOCK_FAST)
|
|
|
1637 goto done;
|
|
|
1638
|
|
|
1639 clock_id = atomic_load_explicit(&fast_clock_id, memory_order_relaxed);
|
|
|
1640 if (clock_id != -1)
|
|
|
1641 goto done;
|
|
|
1642
|
|
|
1643 clock_id = CLOCK_MONOTONIC;
|
|
|
1644 if (0 == clock_getres(CLOCK_MONOTONIC_COARSE, &t))
|
|
|
1645 if (t.tv_nsec <= 1 * 1000 * 1000)
|
|
|
1646 clock_id = CLOCK_MONOTONIC_COARSE;
|
|
|
1647
|
|
|
1648 atomic_store_explicit(&fast_clock_id, clock_id, memory_order_relaxed);
|
|
|
1649
|
|
|
1650 done:
|
|
|
1651
|
|
|
1652 if (clock_gettime(clock_id, &t))
|
|
|
1653 return 0; /* Not really possible. */
|
|
|
1654
|
|
|
1655 return t.tv_sec * (uint64_t) 1e9 + t.tv_nsec;
|
|
|
1656 }
|
|
|
1657
|
|
|
1658
|
|
|
1659 int uv_resident_set_memory(size_t* rss) {
|
|
|
1660 char buf[1024];
|
|
|
1661 const char* s;
|
|
|
1662 long val;
|
|
|
1663 int rc;
|
|
|
1664 int i;
|
|
|
1665
|
|
|
1666 /* rss: 24th element */
|
|
|
1667 rc = uv__slurp("/proc/self/stat", buf, sizeof(buf));
|
|
|
1668 if (rc < 0)
|
|
|
1669 return rc;
|
|
|
1670
|
|
|
1671 /* find the last ')' */
|
|
|
1672 s = strrchr(buf, ')');
|
|
|
1673 if (s == NULL)
|
|
|
1674 goto err;
|
|
|
1675
|
|
|
1676 for (i = 1; i <= 22; i++) {
|
|
|
1677 s = strchr(s + 1, ' ');
|
|
|
1678 if (s == NULL)
|
|
|
1679 goto err;
|
|
|
1680 }
|
|
|
1681
|
|
|
1682 errno = 0;
|
|
|
1683 val = strtol(s, NULL, 10);
|
|
|
1684 if (val < 0 || errno != 0)
|
|
|
1685 goto err;
|
|
|
1686
|
|
|
1687 *rss = val * getpagesize();
|
|
|
1688 return 0;
|
|
|
1689
|
|
|
1690 err:
|
|
|
1691 return UV_EINVAL;
|
|
|
1692 }
|
|
|
1693
|
|
|
1694 int uv_uptime(double* uptime) {
|
|
|
1695 struct timespec now;
|
|
|
1696 char buf[128];
|
|
|
1697
|
|
|
1698 /* Consult /proc/uptime when present (common case), or fall back to
|
|
|
1699 * clock_gettime. Why not always clock_gettime? It doesn't always return the
|
|
|
1700 * right result under OpenVZ and possibly other containerized environments.
|
|
|
1701 */
|
|
|
1702 if (0 == uv__slurp("/proc/uptime", buf, sizeof(buf)))
|
|
|
1703 if (1 == sscanf(buf, "%lf", uptime))
|
|
|
1704 return 0;
|
|
|
1705
|
|
|
1706 if (clock_gettime(CLOCK_BOOTTIME, &now))
|
|
|
1707 return UV__ERR(errno);
|
|
|
1708
|
|
|
1709 *uptime = now.tv_sec;
|
|
|
1710 return 0;
|
|
|
1711 }
|
|
|
1712
|
|
|
1713
|
|
|
1714 int uv_cpu_info(uv_cpu_info_t** ci, int* count) {
|
|
|
1715 #if defined(__PPC__)
|
|
|
1716 static const char model_marker[] = "cpu\t\t: ";
|
|
|
1717 static const char model_marker2[] = "";
|
|
|
1718 #elif defined(__arm__)
|
|
|
1719 static const char model_marker[] = "model name\t: ";
|
|
|
1720 static const char model_marker2[] = "Processor\t: ";
|
|
|
1721 #elif defined(__aarch64__)
|
|
|
1722 static const char model_marker[] = "CPU part\t: ";
|
|
|
1723 static const char model_marker2[] = "";
|
|
|
1724 #elif defined(__mips__)
|
|
|
1725 static const char model_marker[] = "cpu model\t\t: ";
|
|
|
1726 static const char model_marker2[] = "";
|
|
|
1727 #elif defined(__loongarch__)
|
|
|
1728 static const char model_marker[] = "cpu family\t\t: ";
|
|
|
1729 static const char model_marker2[] = "";
|
|
|
1730 #else
|
|
|
1731 static const char model_marker[] = "model name\t: ";
|
|
|
1732 static const char model_marker2[] = "";
|
|
|
1733 #endif
|
|
|
1734 static const char parts[] =
|
|
|
1735 #ifdef __aarch64__
|
|
|
1736 "0x811\nARM810\n" "0x920\nARM920\n" "0x922\nARM922\n"
|
|
|
1737 "0x926\nARM926\n" "0x940\nARM940\n" "0x946\nARM946\n"
|
|
|
1738 "0x966\nARM966\n" "0xa20\nARM1020\n" "0xa22\nARM1022\n"
|
|
|
1739 "0xa26\nARM1026\n" "0xb02\nARM11 MPCore\n" "0xb36\nARM1136\n"
|
|
|
1740 "0xb56\nARM1156\n" "0xb76\nARM1176\n" "0xc05\nCortex-A5\n"
|
|
|
1741 "0xc07\nCortex-A7\n" "0xc08\nCortex-A8\n" "0xc09\nCortex-A9\n"
|
|
|
1742 "0xc0d\nCortex-A17\n" /* Originally A12 */
|
|
|
1743 "0xc0f\nCortex-A15\n" "0xc0e\nCortex-A17\n" "0xc14\nCortex-R4\n"
|
|
|
1744 "0xc15\nCortex-R5\n" "0xc17\nCortex-R7\n" "0xc18\nCortex-R8\n"
|
|
|
1745 "0xc20\nCortex-M0\n" "0xc21\nCortex-M1\n" "0xc23\nCortex-M3\n"
|
|
|
1746 "0xc24\nCortex-M4\n" "0xc27\nCortex-M7\n" "0xc60\nCortex-M0+\n"
|
|
|
1747 "0xd01\nCortex-A32\n" "0xd03\nCortex-A53\n" "0xd04\nCortex-A35\n"
|
|
|
1748 "0xd05\nCortex-A55\n" "0xd06\nCortex-A65\n" "0xd07\nCortex-A57\n"
|
|
|
1749 "0xd08\nCortex-A72\n" "0xd09\nCortex-A73\n" "0xd0a\nCortex-A75\n"
|
|
|
1750 "0xd0b\nCortex-A76\n" "0xd0c\nNeoverse-N1\n" "0xd0d\nCortex-A77\n"
|
|
|
1751 "0xd0e\nCortex-A76AE\n" "0xd13\nCortex-R52\n" "0xd20\nCortex-M23\n"
|
|
|
1752 "0xd21\nCortex-M33\n" "0xd41\nCortex-A78\n" "0xd42\nCortex-A78AE\n"
|
|
|
1753 "0xd4a\nNeoverse-E1\n" "0xd4b\nCortex-A78C\n"
|
|
|
1754 #endif
|
|
|
1755 "";
|
|
|
1756 struct cpu {
|
|
|
1757 unsigned long long freq, user, nice, sys, idle, irq;
|
|
|
1758 unsigned model;
|
|
|
1759 };
|
|
|
1760 FILE* fp;
|
|
|
1761 char* p;
|
|
|
1762 int found;
|
|
|
1763 int n;
|
|
|
1764 unsigned i;
|
|
|
1765 unsigned cpu;
|
|
|
1766 unsigned maxcpu;
|
|
|
1767 unsigned size;
|
|
|
1768 unsigned long long skip;
|
|
|
1769 struct cpu (*cpus)[8192]; /* Kernel maximum. */
|
|
|
1770 struct cpu* c;
|
|
|
1771 struct cpu t;
|
|
|
1772 char (*model)[64];
|
|
|
1773 unsigned char bitmap[ARRAY_SIZE(*cpus) / 8];
|
|
|
1774 /* Assumption: even big.LITTLE systems will have only a handful
|
|
|
1775 * of different CPU models. Most systems will just have one.
|
|
|
1776 */
|
|
|
1777 char models[8][64];
|
|
|
1778 char buf[1024];
|
|
|
1779
|
|
|
1780 memset(bitmap, 0, sizeof(bitmap));
|
|
|
1781 memset(models, 0, sizeof(models));
|
|
|
1782 snprintf(*models, sizeof(*models), "unknown");
|
|
|
1783 maxcpu = 0;
|
|
|
1784
|
|
|
1785 cpus = uv__calloc(ARRAY_SIZE(*cpus), sizeof(**cpus));
|
|
|
1786 if (cpus == NULL)
|
|
|
1787 return UV_ENOMEM;
|
|
|
1788
|
|
|
1789 fp = uv__open_file("/proc/stat");
|
|
|
1790 if (fp == NULL) {
|
|
|
1791 uv__free(cpus);
|
|
|
1792 return UV__ERR(errno);
|
|
|
1793 }
|
|
|
1794
|
|
|
1795 if (NULL == fgets(buf, sizeof(buf), fp))
|
|
|
1796 abort();
|
|
|
1797
|
|
|
1798 for (;;) {
|
|
|
1799 memset(&t, 0, sizeof(t));
|
|
|
1800
|
|
|
1801 n = fscanf(fp, "cpu%u %llu %llu %llu %llu %llu %llu",
|
|
|
1802 &cpu, &t.user, &t.nice, &t.sys, &t.idle, &skip, &t.irq);
|
|
|
1803
|
|
|
1804 if (n != 7)
|
|
|
1805 break;
|
|
|
1806
|
|
|
1807 if (NULL == fgets(buf, sizeof(buf), fp))
|
|
|
1808 abort();
|
|
|
1809
|
|
|
1810 if (cpu >= ARRAY_SIZE(*cpus))
|
|
|
1811 continue;
|
|
|
1812
|
|
|
1813 (*cpus)[cpu] = t;
|
|
|
1814
|
|
|
1815 bitmap[cpu >> 3] |= 1 << (cpu & 7);
|
|
|
1816
|
|
|
1817 if (cpu >= maxcpu)
|
|
|
1818 maxcpu = cpu + 1;
|
|
|
1819 }
|
|
|
1820
|
|
|
1821 fclose(fp);
|
|
|
1822
|
|
|
1823 fp = uv__open_file("/proc/cpuinfo");
|
|
|
1824 if (fp == NULL)
|
|
|
1825 goto nocpuinfo;
|
|
|
1826
|
|
|
1827 for (;;) {
|
|
|
1828 if (1 != fscanf(fp, "processor\t: %u\n", &cpu))
|
|
|
1829 break; /* Parse error. */
|
|
|
1830
|
|
|
1831 while (fgets(buf, sizeof(buf), fp)) {
|
|
|
1832 if (!strncmp(buf, model_marker, sizeof(model_marker) - 1)) {
|
|
|
1833 p = buf + sizeof(model_marker) - 1;
|
|
|
1834 goto parts;
|
|
|
1835 }
|
|
|
1836 if (!*model_marker2)
|
|
|
1837 continue;
|
|
|
1838 if (!strncmp(buf, model_marker2, sizeof(model_marker2) - 1)) {
|
|
|
1839 p = buf + sizeof(model_marker2) - 1;
|
|
|
1840 goto parts;
|
|
|
1841 }
|
|
|
1842 }
|
|
|
1843
|
|
|
1844 goto next; /* Not found. */
|
|
|
1845
|
|
|
1846 parts:
|
|
|
1847 n = (int) strcspn(p, "\n");
|
|
|
1848
|
|
|
1849 /* arm64: translate CPU part code to model name. */
|
|
|
1850 if (*parts) {
|
|
|
1851 p = memmem(parts, sizeof(parts) - 1, p, n + 1);
|
|
|
1852 if (p == NULL)
|
|
|
1853 p = "unknown";
|
|
|
1854 else
|
|
|
1855 p += n + 1;
|
|
|
1856 n = (int) strcspn(p, "\n");
|
|
|
1857 }
|
|
|
1858
|
|
|
1859 found = 0;
|
|
|
1860 for (model = models; !found && model < ARRAY_END(models); model++)
|
|
|
1861 found = !strncmp(p, *model, strlen(*model));
|
|
|
1862
|
|
|
1863 if (!found)
|
|
|
1864 goto next;
|
|
|
1865
|
|
|
1866 if (**model == '\0')
|
|
|
1867 snprintf(*model, sizeof(*model), "%.*s", n, p);
|
|
|
1868
|
|
|
1869 if (cpu < maxcpu)
|
|
|
1870 (*cpus)[cpu].model = model - models;
|
|
|
1871
|
|
|
1872 next:
|
|
|
1873 while (fgets(buf, sizeof(buf), fp))
|
|
|
1874 if (*buf == '\n')
|
|
|
1875 break;
|
|
|
1876 }
|
|
|
1877
|
|
|
1878 fclose(fp);
|
|
|
1879 fp = NULL;
|
|
|
1880
|
|
|
1881 nocpuinfo:
|
|
|
1882
|
|
|
1883 n = 0;
|
|
|
1884 for (cpu = 0; cpu < maxcpu; cpu++) {
|
|
|
1885 if (!(bitmap[cpu >> 3] & (1 << (cpu & 7))))
|
|
|
1886 continue;
|
|
|
1887
|
|
|
1888 n++;
|
|
|
1889 snprintf(buf, sizeof(buf),
|
|
|
1890 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq", cpu);
|
|
|
1891
|
|
|
1892 fp = uv__open_file(buf);
|
|
|
1893 if (fp == NULL)
|
|
|
1894 continue;
|
|
|
1895
|
|
|
1896 if (1 != fscanf(fp, "%llu", &(*cpus)[cpu].freq))
|
|
|
1897 abort();
|
|
|
1898 fclose(fp);
|
|
|
1899 fp = NULL;
|
|
|
1900 }
|
|
|
1901
|
|
|
1902 size = n * sizeof(**ci) + sizeof(models);
|
|
|
1903 *ci = uv__malloc(size);
|
|
|
1904 *count = 0;
|
|
|
1905
|
|
|
1906 if (*ci == NULL) {
|
|
|
1907 uv__free(cpus);
|
|
|
1908 return UV_ENOMEM;
|
|
|
1909 }
|
|
|
1910
|
|
|
1911 *count = n;
|
|
|
1912 p = memcpy(*ci + n, models, sizeof(models));
|
|
|
1913
|
|
|
1914 i = 0;
|
|
|
1915 for (cpu = 0; cpu < maxcpu; cpu++) {
|
|
|
1916 if (!(bitmap[cpu >> 3] & (1 << (cpu & 7))))
|
|
|
1917 continue;
|
|
|
1918
|
|
|
1919 c = *cpus + cpu;
|
|
|
1920
|
|
|
1921 (*ci)[i++] = (uv_cpu_info_t) {
|
|
|
1922 .model = p + c->model * sizeof(*model),
|
|
|
1923 .speed = c->freq / 1000,
|
|
|
1924 /* Note: sysconf(_SC_CLK_TCK) is fixed at 100 Hz,
|
|
|
1925 * therefore the multiplier is always 1000/100 = 10.
|
|
|
1926 */
|
|
|
1927 .cpu_times = (struct uv_cpu_times_s) {
|
|
|
1928 .user = 10 * c->user,
|
|
|
1929 .nice = 10 * c->nice,
|
|
|
1930 .sys = 10 * c->sys,
|
|
|
1931 .idle = 10 * c->idle,
|
|
|
1932 .irq = 10 * c->irq,
|
|
|
1933 },
|
|
|
1934 };
|
|
|
1935 }
|
|
|
1936
|
|
|
1937 uv__free(cpus);
|
|
|
1938
|
|
|
1939 return 0;
|
|
|
1940 }
|
|
|
1941
|
|
|
1942
|
|
|
1943 static int uv__ifaddr_exclude(struct ifaddrs *ent, int exclude_type) {
|
|
|
1944 if (!((ent->ifa_flags & IFF_UP) && (ent->ifa_flags & IFF_RUNNING)))
|
|
|
1945 return 1;
|
|
|
1946 if (ent->ifa_addr == NULL)
|
|
|
1947 return 1;
|
|
|
1948 /*
|
|
|
1949 * On Linux getifaddrs returns information related to the raw underlying
|
|
|
1950 * devices. We're not interested in this information yet.
|
|
|
1951 */
|
|
|
1952 if (ent->ifa_addr->sa_family == PF_PACKET)
|
|
|
1953 return exclude_type;
|
|
|
1954 return !exclude_type;
|
|
|
1955 }
|
|
|
1956
|
|
|
1957 /* TODO(bnoordhuis) share with bsd-ifaddrs.c */
|
|
|
1958 int uv_interface_addresses(uv_interface_address_t** addresses, int* count) {
|
|
|
1959 uv_interface_address_t* address;
|
|
|
1960 struct sockaddr_ll* sll;
|
|
|
1961 struct ifaddrs* addrs;
|
|
|
1962 struct ifaddrs* ent;
|
|
|
1963 size_t namelen;
|
|
|
1964 char* name;
|
|
|
1965 int i;
|
|
|
1966
|
|
|
1967 *count = 0;
|
|
|
1968 *addresses = NULL;
|
|
|
1969
|
|
|
1970 if (getifaddrs(&addrs))
|
|
|
1971 return UV__ERR(errno);
|
|
|
1972
|
|
|
1973 /* Count the number of interfaces */
|
|
|
1974 namelen = 0;
|
|
|
1975 for (ent = addrs; ent != NULL; ent = ent->ifa_next) {
|
|
|
1976 if (uv__ifaddr_exclude(ent, UV__EXCLUDE_IFADDR))
|
|
|
1977 continue;
|
|
|
1978
|
|
|
1979 namelen += strlen(ent->ifa_name) + 1;
|
|
|
1980 (*count)++;
|
|
|
1981 }
|
|
|
1982
|
|
|
1983 if (*count == 0) {
|
|
|
1984 freeifaddrs(addrs);
|
|
|
1985 return 0;
|
|
|
1986 }
|
|
|
1987
|
|
|
1988 /* Make sure the memory is initiallized to zero using calloc() */
|
|
|
1989 *addresses = uv__calloc(1, *count * sizeof(**addresses) + namelen);
|
|
|
1990 if (*addresses == NULL) {
|
|
|
1991 freeifaddrs(addrs);
|
|
|
1992 return UV_ENOMEM;
|
|
|
1993 }
|
|
|
1994
|
|
|
1995 name = (char*) &(*addresses)[*count];
|
|
|
1996 address = *addresses;
|
|
|
1997
|
|
|
1998 for (ent = addrs; ent != NULL; ent = ent->ifa_next) {
|
|
|
1999 if (uv__ifaddr_exclude(ent, UV__EXCLUDE_IFADDR))
|
|
|
2000 continue;
|
|
|
2001
|
|
|
2002 namelen = strlen(ent->ifa_name) + 1;
|
|
|
2003 address->name = memcpy(name, ent->ifa_name, namelen);
|
|
|
2004 name += namelen;
|
|
|
2005
|
|
|
2006 if (ent->ifa_addr->sa_family == AF_INET6) {
|
|
|
2007 address->address.address6 = *((struct sockaddr_in6*) ent->ifa_addr);
|
|
|
2008 } else {
|
|
|
2009 address->address.address4 = *((struct sockaddr_in*) ent->ifa_addr);
|
|
|
2010 }
|
|
|
2011
|
|
|
2012 if (ent->ifa_netmask->sa_family == AF_INET6) {
|
|
|
2013 address->netmask.netmask6 = *((struct sockaddr_in6*) ent->ifa_netmask);
|
|
|
2014 } else {
|
|
|
2015 address->netmask.netmask4 = *((struct sockaddr_in*) ent->ifa_netmask);
|
|
|
2016 }
|
|
|
2017
|
|
|
2018 address->is_internal = !!(ent->ifa_flags & IFF_LOOPBACK);
|
|
|
2019
|
|
|
2020 address++;
|
|
|
2021 }
|
|
|
2022
|
|
|
2023 /* Fill in physical addresses for each interface */
|
|
|
2024 for (ent = addrs; ent != NULL; ent = ent->ifa_next) {
|
|
|
2025 if (uv__ifaddr_exclude(ent, UV__EXCLUDE_IFPHYS))
|
|
|
2026 continue;
|
|
|
2027
|
|
|
2028 address = *addresses;
|
|
|
2029
|
|
|
2030 for (i = 0; i < (*count); i++) {
|
|
|
2031 size_t namelen = strlen(ent->ifa_name);
|
|
|
2032 /* Alias interface share the same physical address */
|
|
|
2033 if (strncmp(address->name, ent->ifa_name, namelen) == 0 &&
|
|
|
2034 (address->name[namelen] == 0 || address->name[namelen] == ':')) {
|
|
|
2035 sll = (struct sockaddr_ll*)ent->ifa_addr;
|
|
|
2036 memcpy(address->phys_addr, sll->sll_addr, sizeof(address->phys_addr));
|
|
|
2037 }
|
|
|
2038 address++;
|
|
|
2039 }
|
|
|
2040 }
|
|
|
2041
|
|
|
2042 freeifaddrs(addrs);
|
|
|
2043
|
|
|
2044 return 0;
|
|
|
2045 }
|
|
|
2046
|
|
|
2047
|
|
|
2048 /* TODO(bnoordhuis) share with bsd-ifaddrs.c */
|
|
|
2049 void uv_free_interface_addresses(uv_interface_address_t* addresses,
|
|
|
2050 int count) {
|
|
|
2051 uv__free(addresses);
|
|
|
2052 }
|
|
|
2053
|
|
|
2054
|
|
|
2055 void uv__set_process_title(const char* title) {
|
|
|
2056 #if defined(PR_SET_NAME)
|
|
|
2057 prctl(PR_SET_NAME, title); /* Only copies first 16 characters. */
|
|
|
2058 #endif
|
|
|
2059 }
|
|
|
2060
|
|
|
2061
|
|
|
2062 static uint64_t uv__read_proc_meminfo(const char* what) {
|
|
|
2063 uint64_t rc;
|
|
|
2064 char* p;
|
|
|
2065 char buf[4096]; /* Large enough to hold all of /proc/meminfo. */
|
|
|
2066
|
|
|
2067 if (uv__slurp("/proc/meminfo", buf, sizeof(buf)))
|
|
|
2068 return 0;
|
|
|
2069
|
|
|
2070 p = strstr(buf, what);
|
|
|
2071
|
|
|
2072 if (p == NULL)
|
|
|
2073 return 0;
|
|
|
2074
|
|
|
2075 p += strlen(what);
|
|
|
2076
|
|
|
2077 rc = 0;
|
|
|
2078 sscanf(p, "%" PRIu64 " kB", &rc);
|
|
|
2079
|
|
|
2080 return rc * 1024;
|
|
|
2081 }
|
|
|
2082
|
|
|
2083
|
|
|
2084 uint64_t uv_get_free_memory(void) {
|
|
|
2085 struct sysinfo info;
|
|
|
2086 uint64_t rc;
|
|
|
2087
|
|
|
2088 rc = uv__read_proc_meminfo("MemAvailable:");
|
|
|
2089
|
|
|
2090 if (rc != 0)
|
|
|
2091 return rc;
|
|
|
2092
|
|
|
2093 if (0 == sysinfo(&info))
|
|
|
2094 return (uint64_t) info.freeram * info.mem_unit;
|
|
|
2095
|
|
|
2096 return 0;
|
|
|
2097 }
|
|
|
2098
|
|
|
2099
|
|
|
2100 uint64_t uv_get_total_memory(void) {
|
|
|
2101 struct sysinfo info;
|
|
|
2102 uint64_t rc;
|
|
|
2103
|
|
|
2104 rc = uv__read_proc_meminfo("MemTotal:");
|
|
|
2105
|
|
|
2106 if (rc != 0)
|
|
|
2107 return rc;
|
|
|
2108
|
|
|
2109 if (0 == sysinfo(&info))
|
|
|
2110 return (uint64_t) info.totalram * info.mem_unit;
|
|
|
2111
|
|
|
2112 return 0;
|
|
|
2113 }
|
|
|
2114
|
|
|
2115
|
|
|
2116 static uint64_t uv__read_uint64(const char* filename) {
|
|
|
2117 char buf[32]; /* Large enough to hold an encoded uint64_t. */
|
|
|
2118 uint64_t rc;
|
|
|
2119
|
|
|
2120 rc = 0;
|
|
|
2121 if (0 == uv__slurp(filename, buf, sizeof(buf)))
|
|
|
2122 if (1 != sscanf(buf, "%" PRIu64, &rc))
|
|
|
2123 if (0 == strcmp(buf, "max\n"))
|
|
|
2124 rc = UINT64_MAX;
|
|
|
2125
|
|
|
2126 return rc;
|
|
|
2127 }
|
|
|
2128
|
|
|
2129
|
|
|
2130 /* Given a buffer with the contents of a cgroup1 /proc/self/cgroups,
|
|
|
2131 * finds the location and length of the memory controller mount path.
|
|
|
2132 * This disregards the leading / for easy concatenation of paths.
|
|
|
2133 * Returns NULL if the memory controller wasn't found. */
|
|
|
2134 static char* uv__cgroup1_find_memory_controller(char buf[static 1024],
|
|
|
2135 int* n) {
|
|
|
2136 char* p;
|
|
|
2137
|
|
|
2138 /* Seek to the memory controller line. */
|
|
|
2139 p = strchr(buf, ':');
|
|
|
2140 while (p != NULL && strncmp(p, ":memory:", 8)) {
|
|
|
2141 p = strchr(p, '\n');
|
|
|
2142 if (p != NULL)
|
|
|
2143 p = strchr(p, ':');
|
|
|
2144 }
|
|
|
2145
|
|
|
2146 if (p != NULL) {
|
|
|
2147 /* Determine the length of the mount path. */
|
|
|
2148 p = p + strlen(":memory:/");
|
|
|
2149 *n = (int) strcspn(p, "\n");
|
|
|
2150 }
|
|
|
2151
|
|
|
2152 return p;
|
|
|
2153 }
|
|
|
2154
|
|
|
2155 static void uv__get_cgroup1_memory_limits(char buf[static 1024], uint64_t* high,
|
|
|
2156 uint64_t* max) {
|
|
|
2157 char filename[4097];
|
|
|
2158 char* p;
|
|
|
2159 int n;
|
|
|
2160 uint64_t cgroup1_max;
|
|
|
2161
|
|
|
2162 /* Find out where the controller is mounted. */
|
|
|
2163 p = uv__cgroup1_find_memory_controller(buf, &n);
|
|
|
2164 if (p != NULL) {
|
|
|
2165 snprintf(filename, sizeof(filename),
|
|
|
2166 "/sys/fs/cgroup/memory/%.*s/memory.soft_limit_in_bytes", n, p);
|
|
|
2167 *high = uv__read_uint64(filename);
|
|
|
2168
|
|
|
2169 snprintf(filename, sizeof(filename),
|
|
|
2170 "/sys/fs/cgroup/memory/%.*s/memory.limit_in_bytes", n, p);
|
|
|
2171 *max = uv__read_uint64(filename);
|
|
|
2172
|
|
|
2173 /* If the controller wasn't mounted, the reads above will have failed,
|
|
|
2174 * as indicated by uv__read_uint64 returning 0.
|
|
|
2175 */
|
|
|
2176 if (*high != 0 && *max != 0)
|
|
|
2177 goto update_limits;
|
|
|
2178 }
|
|
|
2179
|
|
|
2180 /* Fall back to the limits of the global memory controller. */
|
|
|
2181 *high = uv__read_uint64("/sys/fs/cgroup/memory/memory.soft_limit_in_bytes");
|
|
|
2182 *max = uv__read_uint64("/sys/fs/cgroup/memory/memory.limit_in_bytes");
|
|
|
2183
|
|
|
2184 /* uv__read_uint64 detects cgroup2's "max", so we need to separately detect
|
|
|
2185 * cgroup1's maximum value (which is derived from LONG_MAX and PAGE_SIZE).
|
|
|
2186 */
|
|
|
2187 update_limits:
|
|
|
2188 cgroup1_max = LONG_MAX & ~(sysconf(_SC_PAGESIZE) - 1);
|
|
|
2189 if (*high == cgroup1_max)
|
|
|
2190 *high = UINT64_MAX;
|
|
|
2191 if (*max == cgroup1_max)
|
|
|
2192 *max = UINT64_MAX;
|
|
|
2193 }
|
|
|
2194
|
|
|
2195 static void uv__get_cgroup2_memory_limits(char buf[static 1024], uint64_t* high,
|
|
|
2196 uint64_t* max) {
|
|
|
2197 char filename[4097];
|
|
|
2198 char* p;
|
|
|
2199 int n;
|
|
|
2200
|
|
|
2201 /* Find out where the controller is mounted. */
|
|
|
2202 p = buf + strlen("0::/");
|
|
|
2203 n = (int) strcspn(p, "\n");
|
|
|
2204
|
|
|
2205 /* Read the memory limits of the controller. */
|
|
|
2206 snprintf(filename, sizeof(filename), "/sys/fs/cgroup/%.*s/memory.max", n, p);
|
|
|
2207 *max = uv__read_uint64(filename);
|
|
|
2208 snprintf(filename, sizeof(filename), "/sys/fs/cgroup/%.*s/memory.high", n, p);
|
|
|
2209 *high = uv__read_uint64(filename);
|
|
|
2210 }
|
|
|
2211
|
|
|
2212 static uint64_t uv__get_cgroup_constrained_memory(char buf[static 1024]) {
|
|
|
2213 uint64_t high;
|
|
|
2214 uint64_t max;
|
|
|
2215
|
|
|
2216 /* In the case of cgroupv2, we'll only have a single entry. */
|
|
|
2217 if (strncmp(buf, "0::/", 4))
|
|
|
2218 uv__get_cgroup1_memory_limits(buf, &high, &max);
|
|
|
2219 else
|
|
|
2220 uv__get_cgroup2_memory_limits(buf, &high, &max);
|
|
|
2221
|
|
|
2222 if (high == 0 || max == 0)
|
|
|
2223 return 0;
|
|
|
2224
|
|
|
2225 return high < max ? high : max;
|
|
|
2226 }
|
|
|
2227
|
|
|
2228 uint64_t uv_get_constrained_memory(void) {
|
|
|
2229 char buf[1024];
|
|
|
2230
|
|
|
2231 if (uv__slurp("/proc/self/cgroup", buf, sizeof(buf)))
|
|
|
2232 return 0;
|
|
|
2233
|
|
|
2234 return uv__get_cgroup_constrained_memory(buf);
|
|
|
2235 }
|
|
|
2236
|
|
|
2237
|
|
|
2238 static uint64_t uv__get_cgroup1_current_memory(char buf[static 1024]) {
|
|
|
2239 char filename[4097];
|
|
|
2240 uint64_t current;
|
|
|
2241 char* p;
|
|
|
2242 int n;
|
|
|
2243
|
|
|
2244 /* Find out where the controller is mounted. */
|
|
|
2245 p = uv__cgroup1_find_memory_controller(buf, &n);
|
|
|
2246 if (p != NULL) {
|
|
|
2247 snprintf(filename, sizeof(filename),
|
|
|
2248 "/sys/fs/cgroup/memory/%.*s/memory.usage_in_bytes", n, p);
|
|
|
2249 current = uv__read_uint64(filename);
|
|
|
2250
|
|
|
2251 /* If the controller wasn't mounted, the reads above will have failed,
|
|
|
2252 * as indicated by uv__read_uint64 returning 0.
|
|
|
2253 */
|
|
|
2254 if (current != 0)
|
|
|
2255 return current;
|
|
|
2256 }
|
|
|
2257
|
|
|
2258 /* Fall back to the usage of the global memory controller. */
|
|
|
2259 return uv__read_uint64("/sys/fs/cgroup/memory/memory.usage_in_bytes");
|
|
|
2260 }
|
|
|
2261
|
|
|
2262 static uint64_t uv__get_cgroup2_current_memory(char buf[static 1024]) {
|
|
|
2263 char filename[4097];
|
|
|
2264 char* p;
|
|
|
2265 int n;
|
|
|
2266
|
|
|
2267 /* Find out where the controller is mounted. */
|
|
|
2268 p = buf + strlen("0::/");
|
|
|
2269 n = (int) strcspn(p, "\n");
|
|
|
2270
|
|
|
2271 snprintf(filename, sizeof(filename),
|
|
|
2272 "/sys/fs/cgroup/%.*s/memory.current", n, p);
|
|
|
2273 return uv__read_uint64(filename);
|
|
|
2274 }
|
|
|
2275
|
|
|
2276 uint64_t uv_get_available_memory(void) {
|
|
|
2277 char buf[1024];
|
|
|
2278 uint64_t constrained;
|
|
|
2279 uint64_t current;
|
|
|
2280 uint64_t total;
|
|
|
2281
|
|
|
2282 if (uv__slurp("/proc/self/cgroup", buf, sizeof(buf)))
|
|
|
2283 return 0;
|
|
|
2284
|
|
|
2285 constrained = uv__get_cgroup_constrained_memory(buf);
|
|
|
2286 if (constrained == 0)
|
|
|
2287 return uv_get_free_memory();
|
|
|
2288
|
|
|
2289 total = uv_get_total_memory();
|
|
|
2290 if (constrained > total)
|
|
|
2291 return uv_get_free_memory();
|
|
|
2292
|
|
|
2293 /* In the case of cgroupv2, we'll only have a single entry. */
|
|
|
2294 if (strncmp(buf, "0::/", 4))
|
|
|
2295 current = uv__get_cgroup1_current_memory(buf);
|
|
|
2296 else
|
|
|
2297 current = uv__get_cgroup2_current_memory(buf);
|
|
|
2298
|
|
|
2299 /* memory usage can be higher than the limit (for short bursts of time) */
|
|
|
2300 if (constrained < current)
|
|
|
2301 return 0;
|
|
|
2302
|
|
|
2303 return constrained - current;
|
|
|
2304 }
|
|
|
2305
|
|
|
2306
|
|
|
2307 static int uv__get_cgroupv2_constrained_cpu(const char* cgroup,
|
|
|
2308 long long* quota) {
|
|
|
2309 static const char cgroup_mount[] = "/sys/fs/cgroup";
|
|
|
2310 const char* cgroup_trimmed;
|
|
|
2311 char buf[1024];
|
|
|
2312 char full_path[256];
|
|
|
2313 char path[256];
|
|
|
2314 char quota_buf[16];
|
|
|
2315 char* last_slash;
|
|
|
2316 int cgroup_size;
|
|
|
2317 long long limit;
|
|
|
2318 long long min_quota;
|
|
|
2319 long long period;
|
|
|
2320
|
|
|
2321 if (strncmp(cgroup, "0::/", 4) != 0)
|
|
|
2322 return UV_EINVAL;
|
|
|
2323
|
|
|
2324 /* Trim ending \n by replacing it with a 0 */
|
|
|
2325 cgroup_trimmed = cgroup + sizeof("0::/") - 1; /* Skip the prefix "0::/" */
|
|
|
2326 cgroup_size = (int)strcspn(cgroup_trimmed, "\n"); /* Find the first \n */
|
|
|
2327 min_quota = LLONG_MAX;
|
|
|
2328
|
|
|
2329 /* Construct the path to the cpu.max files */
|
|
|
2330 snprintf(path, sizeof(path), "%s/%.*s/cgroup.controllers", cgroup_mount,
|
|
|
2331 cgroup_size, cgroup_trimmed);
|
|
|
2332
|
|
|
2333 /* Read controllers, if not exists, not really a cgroup */
|
|
|
2334 if (uv__slurp(path, buf, sizeof(buf)) < 0)
|
|
|
2335 return UV_EIO;
|
|
|
2336
|
|
|
2337 snprintf(path, sizeof(path), "%s/%.*s", cgroup_mount, cgroup_size,
|
|
|
2338 cgroup_trimmed);
|
|
|
2339
|
|
|
2340 /*
|
|
|
2341 * Traverse up the cgroup v2 hierarchy, starting from the current cgroup path.
|
|
|
2342 * At each level, attempt to read the "cpu.max" file, which defines the CPU
|
|
|
2343 * quota and period.
|
|
|
2344 *
|
|
|
2345 * This reflects how Linux applies cgroup limits hierarchically.
|
|
|
2346 *
|
|
|
2347 * e.g: given a path like /sys/fs/cgroup/foo/bar/baz, we check:
|
|
|
2348 * - /sys/fs/cgroup/foo/bar/baz/cpu.max
|
|
|
2349 * - /sys/fs/cgroup/foo/bar/cpu.max
|
|
|
2350 * - /sys/fs/cgroup/foo/cpu.max
|
|
|
2351 * - /sys/fs/cgroup/cpu.max
|
|
|
2352 */
|
|
|
2353 while (strncmp(path, cgroup_mount, strlen(cgroup_mount)) == 0) {
|
|
|
2354 snprintf(full_path, sizeof(full_path), "%s/cpu.max", path);
|
|
|
2355
|
|
|
2356 /* Silently ignore and continue if the file does not exist */
|
|
|
2357 if (uv__slurp(full_path, quota_buf, sizeof(quota_buf)) < 0)
|
|
|
2358 goto next;
|
|
|
2359
|
|
|
2360 /* No limit, move on */
|
|
|
2361 if (strncmp(quota_buf, "max", 3) == 0)
|
|
|
2362 goto next;
|
|
|
2363
|
|
|
2364 /* Read cpu.max */
|
|
|
2365 if (sscanf(quota_buf, "%lld %lld", &limit, &period) != 2)
|
|
|
2366 goto next;
|
|
|
2367
|
|
|
2368 /* Can't divide by 0 */
|
|
|
2369 if (period == 0)
|
|
|
2370 goto next;
|
|
|
2371
|
|
|
2372 *quota = limit / period;
|
|
|
2373 if (*quota < min_quota)
|
|
|
2374 min_quota = *quota;
|
|
|
2375
|
|
|
2376 next:
|
|
|
2377 /* Move up one level in the cgroup hierarchy by trimming the last path.
|
|
|
2378 * The loop ends once we reach the cgroup root mount point.
|
|
|
2379 */
|
|
|
2380 last_slash = strrchr(path, '/');
|
|
|
2381 if (last_slash == NULL || strcmp(path, cgroup_mount) == 0)
|
|
|
2382 break;
|
|
|
2383 *last_slash = '\0';
|
|
|
2384 }
|
|
|
2385
|
|
|
2386 return 0;
|
|
|
2387 }
|
|
|
2388
|
|
|
2389 static char* uv__cgroup1_find_cpu_controller(const char* cgroup,
|
|
|
2390 int* cgroup_size) {
|
|
|
2391 /* Seek to the cpu controller line. */
|
|
|
2392 char* cgroup_cpu = strstr(cgroup, ":cpu,");
|
|
|
2393
|
|
|
2394 if (cgroup_cpu != NULL) {
|
|
|
2395 /* Skip the controller prefix to the start of the cgroup path. */
|
|
|
2396 cgroup_cpu += sizeof(":cpu,") - 1;
|
|
|
2397 /* Determine the length of the cgroup path, excluding the newline. */
|
|
|
2398 *cgroup_size = (int)strcspn(cgroup_cpu, "\n");
|
|
|
2399 }
|
|
|
2400
|
|
|
2401 return cgroup_cpu;
|
|
|
2402 }
|
|
|
2403
|
|
|
2404 static int uv__get_cgroupv1_constrained_cpu(const char* cgroup,
|
|
|
2405 long long* quota) {
|
|
|
2406 char path[256];
|
|
|
2407 char buf[1024];
|
|
|
2408 int cgroup_size;
|
|
|
2409 char* cgroup_cpu;
|
|
|
2410 long long period_length;
|
|
|
2411 long long quota_per_period;
|
|
|
2412
|
|
|
2413 cgroup_cpu = uv__cgroup1_find_cpu_controller(cgroup, &cgroup_size);
|
|
|
2414
|
|
|
2415 if (cgroup_cpu == NULL)
|
|
|
2416 return UV_EIO;
|
|
|
2417
|
|
|
2418 /* Construct the path to the cpu.cfs_quota_us file */
|
|
|
2419 snprintf(path, sizeof(path), "/sys/fs/cgroup/%.*s/cpu.cfs_quota_us",
|
|
|
2420 cgroup_size, cgroup_cpu);
|
|
|
2421
|
|
|
2422 /* Read cpu.cfs_quota_us */
|
|
|
2423 if (uv__slurp(path, buf, sizeof(buf)) < 0)
|
|
|
2424 return UV_EIO;
|
|
|
2425
|
|
|
2426 if (sscanf(buf, "%lld", "a_per_period) != 1)
|
|
|
2427 return UV_EINVAL;
|
|
|
2428
|
|
|
2429 /* Construct the path to the cpu.cfs_period_us file */
|
|
|
2430 snprintf(path, sizeof(path), "/sys/fs/cgroup/%.*s/cpu.cfs_period_us",
|
|
|
2431 cgroup_size, cgroup_cpu);
|
|
|
2432
|
|
|
2433 /* Read cpu.cfs_period_us */
|
|
|
2434 if (uv__slurp(path, buf, sizeof(buf)) < 0)
|
|
|
2435 return UV_EIO;
|
|
|
2436
|
|
|
2437 if (sscanf(buf, "%lld", &period_length) != 1)
|
|
|
2438 return UV_EINVAL;
|
|
|
2439
|
|
|
2440 /* Can't divide by 0 */
|
|
|
2441 if (period_length == 0)
|
|
|
2442 return UV_EINVAL;
|
|
|
2443
|
|
|
2444 *quota = quota_per_period / period_length;
|
|
|
2445
|
|
|
2446 return 0;
|
|
|
2447 }
|
|
|
2448
|
|
|
2449 int uv__get_constrained_cpu(long long* quota) {
|
|
|
2450 char cgroup[1024];
|
|
|
2451
|
|
|
2452 /* Read the cgroup from /proc/self/cgroup */
|
|
|
2453 if (uv__slurp("/proc/self/cgroup", cgroup, sizeof(cgroup)) < 0)
|
|
|
2454 return UV_EIO;
|
|
|
2455
|
|
|
2456 /* Check if the system is using cgroup v2 by examining /proc/self/cgroup
|
|
|
2457 * The entry for cgroup v2 is always in the format "0::$PATH"
|
|
|
2458 * see https://docs.kernel.org/admin-guide/cgroup-v2.html */
|
|
|
2459 if (strncmp(cgroup, "0::/", 4) == 0)
|
|
|
2460 return uv__get_cgroupv2_constrained_cpu(cgroup, quota);
|
|
|
2461 else
|
|
|
2462 return uv__get_cgroupv1_constrained_cpu(cgroup, quota);
|
|
|
2463 }
|
|
|
2464
|
|
|
2465
|
|
|
2466 void uv_loadavg(double avg[3]) {
|
|
|
2467 struct sysinfo info;
|
|
|
2468 char buf[128]; /* Large enough to hold all of /proc/loadavg. */
|
|
|
2469
|
|
|
2470 if (0 == uv__slurp("/proc/loadavg", buf, sizeof(buf)))
|
|
|
2471 if (3 == sscanf(buf, "%lf %lf %lf", &avg[0], &avg[1], &avg[2]))
|
|
|
2472 return;
|
|
|
2473
|
|
|
2474 if (sysinfo(&info) < 0)
|
|
|
2475 return;
|
|
|
2476
|
|
|
2477 avg[0] = (double) info.loads[0] / 65536.0;
|
|
|
2478 avg[1] = (double) info.loads[1] / 65536.0;
|
|
|
2479 avg[2] = (double) info.loads[2] / 65536.0;
|
|
|
2480 }
|
|
|
2481
|
|
|
2482
|
|
|
2483 static int compare_watchers(const struct watcher_list* a,
|
|
|
2484 const struct watcher_list* b) {
|
|
|
2485 if (a->wd < b->wd) return -1;
|
|
|
2486 if (a->wd > b->wd) return 1;
|
|
|
2487 return 0;
|
|
|
2488 }
|
|
|
2489
|
|
|
2490
|
|
|
2491 static int init_inotify(uv_loop_t* loop) {
|
|
|
2492 int err;
|
|
|
2493 int fd;
|
|
|
2494
|
|
|
2495 if (loop->inotify_fd != -1)
|
|
|
2496 return 0;
|
|
|
2497
|
|
|
2498 fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC);
|
|
|
2499 if (fd < 0)
|
|
|
2500 return UV__ERR(errno);
|
|
|
2501
|
|
|
2502 err = uv__io_init_start(loop, &loop->inotify_read_watcher, uv__inotify_read,
|
|
|
2503 fd, POLLIN);
|
|
|
2504 if (err) {
|
|
|
2505 uv__close(fd);
|
|
|
2506 return err;
|
|
|
2507 }
|
|
|
2508
|
|
|
2509 loop->inotify_fd = fd;
|
|
|
2510 return 0;
|
|
|
2511 }
|
|
|
2512
|
|
|
2513
|
|
|
2514 static int uv__inotify_fork(uv_loop_t* loop, struct watcher_list* root) {
|
|
|
2515 /* Open the inotify_fd, and re-arm all the inotify watchers. */
|
|
|
2516 int err;
|
|
|
2517 struct watcher_list* tmp_watcher_list_iter;
|
|
|
2518 struct watcher_list* watcher_list;
|
|
|
2519 struct watcher_list tmp_watcher_list;
|
|
|
2520 struct uv__queue queue;
|
|
|
2521 struct uv__queue* q;
|
|
|
2522 uv_fs_event_t* handle;
|
|
|
2523 char* tmp_path;
|
|
|
2524
|
|
|
2525 if (root == NULL)
|
|
|
2526 return 0;
|
|
|
2527
|
|
|
2528 /* We must restore the old watcher list to be able to close items
|
|
|
2529 * out of it.
|
|
|
2530 */
|
|
|
2531 loop->inotify_watchers = root;
|
|
|
2532
|
|
|
2533 uv__queue_init(&tmp_watcher_list.watchers);
|
|
|
2534 /* Note that the queue we use is shared with the start and stop()
|
|
|
2535 * functions, making uv__queue_foreach unsafe to use. So we use the
|
|
|
2536 * uv__queue_move trick to safely iterate. Also don't free the watcher
|
|
|
2537 * list until we're done iterating. c.f. uv__inotify_read.
|
|
|
2538 */
|
|
|
2539 RB_FOREACH_SAFE(watcher_list, watcher_root,
|
|
|
2540 uv__inotify_watchers(loop), tmp_watcher_list_iter) {
|
|
|
2541 watcher_list->iterating = 1;
|
|
|
2542 uv__queue_move(&watcher_list->watchers, &queue);
|
|
|
2543 while (!uv__queue_empty(&queue)) {
|
|
|
2544 q = uv__queue_head(&queue);
|
|
|
2545 handle = uv__queue_data(q, uv_fs_event_t, watchers);
|
|
|
2546 /* It's critical to keep a copy of path here, because it
|
|
|
2547 * will be set to NULL by stop() and then deallocated by
|
|
|
2548 * maybe_free_watcher_list
|
|
|
2549 */
|
|
|
2550 tmp_path = uv__strdup(handle->path);
|
|
|
2551 assert(tmp_path != NULL);
|
|
|
2552 uv__queue_remove(q);
|
|
|
2553 uv__queue_insert_tail(&watcher_list->watchers, q);
|
|
|
2554 uv_fs_event_stop(handle);
|
|
|
2555
|
|
|
2556 uv__queue_insert_tail(&tmp_watcher_list.watchers, &handle->watchers);
|
|
|
2557 handle->path = tmp_path;
|
|
|
2558 }
|
|
|
2559 watcher_list->iterating = 0;
|
|
|
2560 maybe_free_watcher_list(watcher_list, loop);
|
|
|
2561 }
|
|
|
2562
|
|
|
2563 uv__queue_move(&tmp_watcher_list.watchers, &queue);
|
|
|
2564 while (!uv__queue_empty(&queue)) {
|
|
|
2565 q = uv__queue_head(&queue);
|
|
|
2566 uv__queue_remove(q);
|
|
|
2567 handle = uv__queue_data(q, uv_fs_event_t, watchers);
|
|
|
2568 tmp_path = handle->path;
|
|
|
2569 handle->path = NULL;
|
|
|
2570 err = uv_fs_event_start(handle, handle->cb, tmp_path, 0);
|
|
|
2571 uv__free(tmp_path);
|
|
|
2572 if (err)
|
|
|
2573 return err;
|
|
|
2574 }
|
|
|
2575
|
|
|
2576 return 0;
|
|
|
2577 }
|
|
|
2578
|
|
|
2579
|
|
|
2580 static struct watcher_list* find_watcher(uv_loop_t* loop, int wd) {
|
|
|
2581 struct watcher_list w;
|
|
|
2582 w.wd = wd;
|
|
|
2583 return RB_FIND(watcher_root, uv__inotify_watchers(loop), &w);
|
|
|
2584 }
|
|
|
2585
|
|
|
2586
|
|
|
2587 static void maybe_free_watcher_list(struct watcher_list* w, uv_loop_t* loop) {
|
|
|
2588 /* if the watcher_list->watchers is being iterated over, we can't free it. */
|
|
|
2589 if ((!w->iterating) && uv__queue_empty(&w->watchers)) {
|
|
|
2590 /* No watchers left for this path. Clean up. */
|
|
|
2591 RB_REMOVE(watcher_root, uv__inotify_watchers(loop), w);
|
|
|
2592 inotify_rm_watch(loop->inotify_fd, w->wd);
|
|
|
2593 uv__free(w);
|
|
|
2594 }
|
|
|
2595 }
|
|
|
2596
|
|
|
2597
|
|
|
2598 static void uv__inotify_read(uv_loop_t* loop,
|
|
|
2599 uv__io_t* dummy,
|
|
|
2600 unsigned int events) {
|
|
|
2601 const struct inotify_event* e;
|
|
|
2602 struct watcher_list* w;
|
|
|
2603 uv_fs_event_t* h;
|
|
|
2604 struct uv__queue queue;
|
|
|
2605 struct uv__queue* q;
|
|
|
2606 const char* path;
|
|
|
2607 ssize_t size;
|
|
|
2608 const char *p;
|
|
|
2609 /* needs to be large enough for sizeof(inotify_event) + strlen(path) */
|
|
|
2610 char buf[4096];
|
|
|
2611
|
|
|
2612 for (;;) {
|
|
|
2613 do
|
|
|
2614 size = read(loop->inotify_fd, buf, sizeof(buf));
|
|
|
2615 while (size == -1 && errno == EINTR);
|
|
|
2616
|
|
|
2617 if (size == -1) {
|
|
|
2618 assert(errno == EAGAIN || errno == EWOULDBLOCK);
|
|
|
2619 break;
|
|
|
2620 }
|
|
|
2621
|
|
|
2622 assert(size > 0); /* pre-2.6.21 thing, size=0 == read buffer too small */
|
|
|
2623
|
|
|
2624 /* Now we have one or more inotify_event structs. */
|
|
|
2625 for (p = buf; p < buf + size; p += sizeof(*e) + e->len) {
|
|
|
2626 e = (const struct inotify_event*) p;
|
|
|
2627
|
|
|
2628 events = 0;
|
|
|
2629 if (e->mask & (IN_ATTRIB|IN_MODIFY))
|
|
|
2630 events |= UV_CHANGE;
|
|
|
2631 if (e->mask & ~(IN_ATTRIB|IN_MODIFY))
|
|
|
2632 events |= UV_RENAME;
|
|
|
2633
|
|
|
2634 w = find_watcher(loop, e->wd);
|
|
|
2635 if (w == NULL)
|
|
|
2636 continue; /* Stale event, no watchers left. */
|
|
|
2637
|
|
|
2638 /* inotify does not return the filename when monitoring a single file
|
|
|
2639 * for modifications. Repurpose the filename for API compatibility.
|
|
|
2640 * I'm not convinced this is a good thing, maybe it should go.
|
|
|
2641 */
|
|
|
2642 path = e->len ? (const char*) (e + 1) : uv__basename_r(w->path);
|
|
|
2643
|
|
|
2644 /* We're about to iterate over the queue and call user's callbacks.
|
|
|
2645 * What can go wrong?
|
|
|
2646 * A callback could call uv_fs_event_stop()
|
|
|
2647 * and the queue can change under our feet.
|
|
|
2648 * So, we use uv__queue_move() trick to safely iterate over the queue.
|
|
|
2649 * And we don't free the watcher_list until we're done iterating.
|
|
|
2650 *
|
|
|
2651 * First,
|
|
|
2652 * tell uv_fs_event_stop() (that could be called from a user's callback)
|
|
|
2653 * not to free watcher_list.
|
|
|
2654 */
|
|
|
2655 w->iterating = 1;
|
|
|
2656 uv__queue_move(&w->watchers, &queue);
|
|
|
2657 while (!uv__queue_empty(&queue)) {
|
|
|
2658 q = uv__queue_head(&queue);
|
|
|
2659 h = uv__queue_data(q, uv_fs_event_t, watchers);
|
|
|
2660
|
|
|
2661 uv__queue_remove(q);
|
|
|
2662 uv__queue_insert_tail(&w->watchers, q);
|
|
|
2663
|
|
|
2664 h->cb(h, path, events, 0);
|
|
|
2665 }
|
|
|
2666 /* done iterating, time to (maybe) free empty watcher_list */
|
|
|
2667 w->iterating = 0;
|
|
|
2668 maybe_free_watcher_list(w, loop);
|
|
|
2669 }
|
|
|
2670 }
|
|
|
2671 }
|
|
|
2672
|
|
|
2673
|
|
|
2674 int uv_fs_event_init(uv_loop_t* loop, uv_fs_event_t* handle) {
|
|
|
2675 uv__handle_init(loop, (uv_handle_t*)handle, UV_FS_EVENT);
|
|
|
2676 return 0;
|
|
|
2677 }
|
|
|
2678
|
|
|
2679
|
|
|
2680 int uv_fs_event_start(uv_fs_event_t* handle,
|
|
|
2681 uv_fs_event_cb cb,
|
|
|
2682 const char* path,
|
|
|
2683 unsigned int flags) {
|
|
|
2684 struct watcher_list* w;
|
|
|
2685 uv_loop_t* loop;
|
|
|
2686 size_t len;
|
|
|
2687 int events;
|
|
|
2688 int err;
|
|
|
2689 int wd;
|
|
|
2690
|
|
|
2691 if (uv__is_active(handle))
|
|
|
2692 return UV_EINVAL;
|
|
|
2693
|
|
|
2694 loop = handle->loop;
|
|
|
2695
|
|
|
2696 err = init_inotify(loop);
|
|
|
2697 if (err)
|
|
|
2698 return err;
|
|
|
2699
|
|
|
2700 events = IN_ATTRIB
|
|
|
2701 | IN_CREATE
|
|
|
2702 | IN_MODIFY
|
|
|
2703 | IN_DELETE
|
|
|
2704 | IN_DELETE_SELF
|
|
|
2705 | IN_MOVE_SELF
|
|
|
2706 | IN_MOVED_FROM
|
|
|
2707 | IN_MOVED_TO;
|
|
|
2708
|
|
|
2709 wd = inotify_add_watch(loop->inotify_fd, path, events);
|
|
|
2710 if (wd == -1)
|
|
|
2711 return UV__ERR(errno);
|
|
|
2712
|
|
|
2713 w = find_watcher(loop, wd);
|
|
|
2714 if (w)
|
|
|
2715 goto no_insert;
|
|
|
2716
|
|
|
2717 len = strlen(path) + 1;
|
|
|
2718 w = uv__malloc(sizeof(*w) + len);
|
|
|
2719 if (w == NULL)
|
|
|
2720 return UV_ENOMEM;
|
|
|
2721
|
|
|
2722 w->wd = wd;
|
|
|
2723 w->path = memcpy(w + 1, path, len);
|
|
|
2724 uv__queue_init(&w->watchers);
|
|
|
2725 w->iterating = 0;
|
|
|
2726 RB_INSERT(watcher_root, uv__inotify_watchers(loop), w);
|
|
|
2727
|
|
|
2728 no_insert:
|
|
|
2729 uv__handle_start(handle);
|
|
|
2730 uv__queue_insert_tail(&w->watchers, &handle->watchers);
|
|
|
2731 handle->path = w->path;
|
|
|
2732 handle->cb = cb;
|
|
|
2733 handle->wd = wd;
|
|
|
2734
|
|
|
2735 return 0;
|
|
|
2736 }
|
|
|
2737
|
|
|
2738
|
|
|
2739 int uv_fs_event_stop(uv_fs_event_t* handle) {
|
|
|
2740 struct watcher_list* w;
|
|
|
2741
|
|
|
2742 if (!uv__is_active(handle))
|
|
|
2743 return 0;
|
|
|
2744
|
|
|
2745 w = find_watcher(handle->loop, handle->wd);
|
|
|
2746 assert(w != NULL);
|
|
|
2747
|
|
|
2748 handle->wd = -1;
|
|
|
2749 handle->path = NULL;
|
|
|
2750 uv__handle_stop(handle);
|
|
|
2751 uv__queue_remove(&handle->watchers);
|
|
|
2752
|
|
|
2753 maybe_free_watcher_list(w, handle->loop);
|
|
|
2754
|
|
|
2755 return 0;
|
|
|
2756 }
|
|
|
2757
|
|
|
2758
|
|
|
2759 void uv__fs_event_close(uv_fs_event_t* handle) {
|
|
|
2760 uv_fs_event_stop(handle);
|
|
|
2761 }
|
