1 /*******************************************************************************
2 
3     Manages an I/O event loop with automatic handler invocation and
4     unregistration.
5 
6     The EpollSelectDispatcher class wraps a Tango EpollSelector and uses
7     ISelectClient instances for Select I/O event registration, unregistration
8     and event handler invocation. An I/O event loop is provided that runs while
9     there are select event registrations. This loop automatically invokes the
10     registered handlers; via the return value each handler may indicate that it
11     wishes to be unregistered. After the ISelectClient instance has been
12     unregistered, its finalize() method is invoked.
13 
14     If a handler throws an Exception, it is caught, the ISelectClient containing
15     that handler is unregistered immediately and finalize() is invoked.
16     Exceptions thrown by the ISelectClient's finalize() methods are also caught.
17 
18     Copyright:
19         Copyright (c) 2009-2016 dunnhumby Germany GmbH.
20         All rights reserved.
21 
22     License:
23         Boost Software License Version 1.0. See LICENSE_BOOST.txt for details.
24         Alternatively, this file may be distributed under the terms of the Tango
25         3-Clause BSD License (see LICENSE_BSD.txt for details).
26 
27 *******************************************************************************/
28 
29 module ocean.io.select.EpollSelectDispatcher;
30 
31 
32 import ocean.meta.types.Qualifiers;
33 
34 import ocean.core.Verify;
35 
36 import ocean.io.select.selector.IEpollSelectDispatcherInfo;
37 import ocean.io.select.client.model.ISelectClient;
38 
39 import ocean.io.select.selector.model.ISelectedKeysHandler;
40 
41 import ocean.io.select.selector.SelectedKeysHandler,
42        ocean.io.select.selector.TimeoutSelectedKeysHandler,
43        ocean.io.select.selector.EpollException;
44 
45 import ocean.util.ReusableException;
46 
47 import ocean.io.select.client.SelectEvent;
48 
49 import ocean.io.select.selector.RegisteredClients;
50 
51 import ocean.core.Array : copy;
52 
53 import ocean.core.array.Search;
54 
55 import ocean.util.container.AppendBuffer;
56 
57 import ocean.util.container.queue.DynamicQueue;
58 
59 import ocean.time.timeout.model.ITimeoutManager;
60 
61 import ocean.sys.Epoll;
62 
63 import core.stdc.stdlib: bsearch, qsort;
64 
65 import core.stdc.errno: errno, EINTR, ENOENT, EEXIST, ENOMEM, EINVAL;
66 
67 debug ( ISelectClient ) import ocean.io.Stdout;
68 
69 version (unittest)
70 {
71     debug = EpollFdSanity;
72 }
73 
74 debug (EpollFdSanity)
75 {
76     import ocean.io.select.selector.EpollFdSanity;
77 }
78 
79 /*******************************************************************************
80 
81     EpollSelectDispatcher
82 
83 *******************************************************************************/
84 
85 public class EpollSelectDispatcher : IEpollSelectDispatcherInfo
86 {
87     /***************************************************************************
88 
89         Event alias used internally
90 
91      **************************************************************************/
92 
93     alias ISelectClient.Event Event;
94 
95     /***************************************************************************
96 
97         Delegate to be called each time select cycle finished.
98 
99      **************************************************************************/
100 
101     alias void delegate () SelectCycleCallback;
102 
103     /***************************************************************************
104 
105         Set of registered clients.
106 
107      **************************************************************************/
108 
109     private IRegisteredClients registered_clients;
110 
111     /**************************************************************************
112 
113          Default maximum number of file descriptors for which events can be
114          reported with one epoll_wait() call.
115 
116      **************************************************************************/
117 
118     public static immutable uint DefaultMaxEvents = 16;
119 
120     /**************************************************************************
121 
122          true if the timeout feature is enabled.
123 
124      **************************************************************************/
125 
126     public bool timeout_enabled ( )
127     {
128         return this._timeout_enabled;
129     }
130 
131     private bool _timeout_enabled;
132 
133     /***************************************************************************
134 
135         Wrapped Epoll file handle
136 
137      **************************************************************************/
138 
139     private Epoll epoll;
140 
141     /***************************************************************************
142 
143         Reused list of events.
144 
145      **************************************************************************/
146 
147     private epoll_event_t[] events;
148 
149     /***************************************************************************
150 
151         List of the clients that we're currently handling. Always slices
152         this.events.
153 
154     ***************************************************************************/
155 
156     private epoll_event_t[] selected_set;
157 
158     /***************************************************************************
159 
160         Queue of delegates to be called (and dismissed) after the
161         epoll select cycle finishes.
162 
163     ***************************************************************************/
164 
165     private DynamicQueue!(SelectCycleCallback) select_cycle_callbacks;
166 
167     /***************************************************************************
168 
169         Re-usable errno exception
170 
171      **************************************************************************/
172 
173     private EpollException    e;
174 
175     /***************************************************************************
176 
177         Optional hook to be called on unhandled exceptions from event callbacks.
178 
179         NB: it won't be called on actual event errors thrown as EpollException
180         as those are expected to be handled on select client level exclusively.
181 
182     ***************************************************************************/
183 
184     private bool delegate (Exception) unhandled_exception_hook;
185 
186     /***************************************************************************
187 
188         Timeout manager instance; null disables the timeout feature.
189 
190      **************************************************************************/
191 
192     private ITimeoutManager timeout_manager;
193 
194     /***************************************************************************
195 
196         Event which is triggered when the shutdown() method is called.
197 
198      **************************************************************************/
199 
200     private SelectEvent shutdown_event;
201 
202     /***************************************************************************
203 
204         Flag which the eventLoop checks for exit status. Set to true when the
205         shutdown event fires (via calling the shutdown() method).
206 
207      **************************************************************************/
208 
209     private bool shutdown_triggered;
210 
211     /***************************************************************************
212 
213         Flag set to true when the eventLoop() method is called, and to false
214         when it exits. Used to assert that the event loop is not started from
215         within itself.
216 
217      **************************************************************************/
218 
219     private bool in_event_loop;
220 
221     version ( EpollCounters )
222     {
223         /***********************************************************************
224 
225             Struct containing counters to track stats about the selector.
226 
227         ***********************************************************************/
228 
229         private struct Counters
230         {
231             ulong selects;
232             ulong timeouts;
233         }
234 
235         private Counters counters;
236     }
237 
238     /***************************************************************************
239 
240         Client handler.
241 
242      **************************************************************************/
243 
244     private ISelectedKeysHandler handle;
245 
246     /***************************************************************************
247 
248         Constructor
249 
250         Params:
251             timeout_manager = timeout manager instance (null disables the
252                               timeout feature)
253             max_events      = sets the maximum number of events that will be
254                               returned in the selection set per call to select.
255 
256         Throws:
257             EpollException on error obtaining a new epoll instance.
258 
259      **************************************************************************/
260 
261     public this ( ITimeoutManager timeout_manager = null, uint max_events = DefaultMaxEvents )
262     {
263         debug ( ISelectClient )
264         {
265             this.registered_clients = new ClientSet;
266         }
267         else
268         {
269             this.registered_clients = new ClientCount;
270         }
271 
272         this.e = new EpollException;
273 
274         this.e.enforce(
275             this.epoll.create() >= 0,
276             "error creating epoll object",
277             "epoll_create"
278         );
279 
280         this.timeout_manager = timeout_manager;
281 
282         this._timeout_enabled = timeout_manager !is null;
283 
284         this.shutdown_event = new SelectEvent(&this.shutdownTrigger);
285 
286         this.events            = new epoll_event_t[max_events];
287 
288         this.handle = this.timeout_enabled?
289             new TimeoutSelectedKeysHandler(&this.unregister, this.e, this.timeout_manager, max_events) :
290             new SelectedKeysHandler(&this.unregister, this.e);
291 
292         this.select_cycle_callbacks = new DynamicQueue!(SelectCycleCallback);
293         this.select_cycle_callbacks.auto_shrink = false;
294     }
295 
296     /***************************************************************************
297 
298         Constructor; disables the timeout feature.
299 
300         Params:
301             max_events      = sets the maximum number of events that will be
302                               returned in the selection set per call to select.
303 
304      **************************************************************************/
305 
306     public this ( uint max_events )
307     {
308         this(null, max_events);
309     }
310 
311     /***************************************************************************
312 
313         Destructor.
314 
315      **************************************************************************/
316 
317     ~this ( )
318     {
319         with (this.epoll) if (fd >= 0)
320         {
321             close();
322         }
323     }
324 
325     /***************************************************************************
326 
327         Adds new callback to the queue of delegates to be called after the
328         current select cycle. It will only be kept for a single cycle.
329 
330         Params:
331             cb = callback to call once after the current cycle ends. Must return
332                 `true` to block event loop from terminating in the current
333                 cycle. Returning `false` is primarily intended for internal
334                 epoll/scheduler facilities and unlikely to be of use in an
335                 application code.
336 
337     ***************************************************************************/
338 
339     public void onCycleEnd ( scope SelectCycleCallback cb )
340     {
341         this.select_cycle_callbacks.push(cb);
342     }
343 
344     /***************************************************************************
345 
346         Adds or modifies a client registration.
347 
348         To change the client of a currently registered conduit when several
349         clients share the same conduit, use changeClient().
350 
351         Important note: client is stored in a memory location not managed by the
352         D runtime memory manager (aka Garbage Collector). Therefore it is
353         important that the caller makes sure client is stored somewhere visible
354         to the GC (in a class variable, for example) so it won't get garbage
355         collected and deleted.
356 
357         Params:
358             client = client to register, please make sure it is stored somewhere
359                      visible to the garbage collector
360 
361         Returns:
362             true if everything worked as expected or false if the client was
363             unexpectedly unregistered as it happens when its file descriptor is
364             closed.
365 
366         Throws:
367             EpollException on error.
368 
369      **************************************************************************/
370 
371     public bool register ( ISelectClient client )
372     {
373         try if (client.is_registered)
374         {
375             scope (failure)
376             {
377                 this.registered_clients -= client;
378             }
379 
380             return this.modify(client);
381         }
382         else
383         {
384             this.e.enforce(
385                 this.epollCtl(epoll.CtlOp.EPOLL_CTL_ADD, client.fileHandle,
386                     client.events, client) == 0,
387                 "error adding epoll registration",
388                 "epoll_ctl"
389             );
390 
391             this.registered_clients += client;
392 
393             return true;
394         }
395         catch (Exception e)
396         {
397             debug ( ISelectClient )
398             {
399                 Stderr.formatln("{} :: Error during register: '{}' @{}:{}",
400                     client, e.message(), e.file, e.line).flush();
401             }
402             throw e;
403         }
404     }
405 
406     /***************************************************************************
407 
408        Removes a client registration. Does not fail/throw if the client is not
409        registered.
410 
411        Params:
412             client = client to unregister
413             remove_from_selected_set = if true, removes the client from the
414                 selected set that may be currently being iterated over. This
415                 guarantees that the unregistered client's handle method will not
416                 be subsequently called by the selector. The client may thus be
417                 safely destroyed after unregistering.
418 
419        Returns:
420             0 if everything worked as expected or the error code (errno) as a
421             warning on minor errors, that is, everything except ENOMEM (out of
422             memory) and EINVAL (invalid epoll file descriptor or epoll_ctl()
423             opcode).
424             ENOENT is a minor error that happens regularly when the client was
425             unexpectedly unregistered as it happens when its file descriptor is
426             closed.
427 
428         Throws:
429             EpollException on the fatal errors ENOMEM and EINVAL.
430 
431      **************************************************************************/
432 
433     public int unregister ( ISelectClient client,
434         bool remove_from_selected_set = false )
435     {
436         try if (client.is_registered)
437         {
438             scope (success)
439             {
440                 this.registered_clients -= client;
441 
442                 if (remove_from_selected_set)
443                 {
444                     /// Predicate for finding the ISelectClient inside
445                     /// array of epoll_event_t entries.
446                     scope entry_to_client = (const(epoll_event_t) entry) {
447                         return entry.data.ptr == cast(void*)client;
448                     };
449 
450                     auto index = this.selected_set.findIf(entry_to_client);
451                     // Instead of removing the array entry, we'll just invalidate
452                     // it. This is to avoid problems with both the fact that
453                     // SelectedKeysHandler might be foreach-iterating over
454                     // this array at this time and the fact that shrinking
455                     // the slice owned by EpollSelectDispatcher wouldn't shrink
456                     // the slices owned by SelectedKeysHandler.
457                     if (index < this.selected_set.length)
458                     {
459                         this.selected_set[index] = epoll_event_t.init;
460                     }
461                 }
462             }
463 
464             if (!this.epollCtl(epoll.CtlOp.EPOLL_CTL_DEL, client.fileHandle,
465                 client.events, client))
466             {
467                 return 0;
468             }
469             else
470             {
471                 int errnum = .errno;
472 
473                 switch (errnum)
474                 {
475                     default:
476                         return errnum;
477 
478                     case ENOMEM, EINVAL:
479                         throw this.e.set(errnum)
480                             .addMessage("error removing epoll client");
481                 }
482             }
483         }
484         else
485         {
486             return false;
487         }
488         catch (Exception e)
489         {
490             debug ( ISelectClient )
491             {
492                 Stderr.formatln("{} :: Error during unregister: '{}' @{}:{}",
493                     client, e.message(), e.file, e.line).flush();
494             }
495             throw e;
496         }
497     }
498 
499     /**************************************************************************
500 
501         Changes the clients of a registered conduit from current to next.
502 
503         - current and next are expected to share the the same file descriptor
504           (conduit file handle),
505         - current is expected to be registered while next is expected not to be
506           registered. It is tolerated if current is unexpectedly unregistered
507           as it happens when its file descriptor is closed.
508 
509        Important note: next is stored in a memory location not managed by the
510        D runtime memory manager (aka Garbage Collector). Therefore it is
511        important that the caller makes sure next is stored somewhere visible
512        to the GC (in a class variable, for example) so it won't get garbage
513        collected and deleted.
514 
515        Params:
516             current = currently registered client to be unregistered
517             next    = currently unregistered client to be registered, please
518                       make sure it is stored somewhere visible to the garbage
519                       collector
520 
521        Returns:
522             true if everything worked as expected or false if current is
523             unexpectedly unregistered as it happens when its file descriptor is
524             closed.
525 
526         Throws:
527             EpollException on error.
528 
529         In:
530             - current and next must have the the same file descriptor,
531             - current.is_registered must be true,
532             - next.is_registered must be false.
533 
534      **************************************************************************/
535 
536     public bool changeClient ( ISelectClient current, ISelectClient next )
537     {
538         debug ( ISelectClient )
539         {
540             if (current.fileHandle != next.fileHandle)
541             {
542                 Stderr.formatln("Error during changeClient: current.fileHandle != next.fileHandle").flush();
543             }
544 
545             if (!current.is_registered)
546             {
547                 Stderr.formatln("Error during changeClient: !current.is_registered").flush();
548             }
549 
550             if (next.is_registered)
551             {
552                 Stderr.formatln("Error during changeClient: next.is_registered").flush();
553             }
554         }
555 
556         verify(current.fileHandle == next.fileHandle,
557                 typeof (this).stringof ~ ".changeClient: clients are expected to share the same file descriptor");
558         verify(current.is_registered,
559                 typeof (this).stringof ~ ".changeClient: current client is expected to be registered");
560         verify(!next.is_registered,
561                 typeof (this).stringof ~ ".changeClient: next client is expected not to be registered");
562         verify(current !is next); // should be impossible since current.is_registered != next.is_registered
563 
564         try
565         {
566             scope (success)
567             {
568                 debug ( ISelectClient )
569                 {
570                     Stderr.formatln("Changed clients for fd:").flush();
571                     Stderr.formatln("  Replaced {}", current).flush();
572                     Stderr.formatln("  with     {}", next).flush();
573                 }
574 
575                 this.registered_clients -= current;
576                 this.registered_clients += next;
577             }
578 
579             return this.modify(next);
580         }
581         catch (Exception e)
582         {
583             debug ( ISelectClient )
584             {
585                 Stderr.formatln("Error during changeClient: '{}' @{}:{}",
586                     e.message(), e.file, e.line).flush();
587             }
588             throw e;
589         }
590     }
591 
592     /**************************************************************************
593 
594         IEpollSelectDispatcherInfo interface method.
595 
596         Returns:
597             the number of clients registered with the select dispatcher
598 
599      **************************************************************************/
600 
601     public size_t num_registered ( )
602     {
603         return this.registered_clients.length;
604     }
605 
606     version ( EpollCounters )
607     {
608         /***********************************************************************
609 
610             Returns:
611                 the number of select calls (epoll_wait()) since the instance was
612                 created (or since the ulong counter wrapped)
613 
614         ***********************************************************************/
615 
616         public ulong selects ( )
617         {
618             return this.counters.selects;
619         }
620 
621 
622         /***********************************************************************
623 
624             Returns:
625                 the number of select calls (epoll_wait()) which exited due to a
626                 timeout (as opposed to a client firing) since the instance was
627                 created (or since the ulong counter wrapped)
628 
629         ***********************************************************************/
630 
631         public ulong timeouts ( )
632         {
633             return this.counters.timeouts;
634         }
635 
636 
637         /***********************************************************************
638 
639             Resets the counters returned by selects() and timeouts().
640 
641         ***********************************************************************/
642 
643         public void resetCounters ( )
644         {
645             this.counters = this.counters.init;
646         }
647     }
648 
649     /**************************************************************************
650 
651         Modifies the registration of client using EPOLL_CTL_MOD.
652         More precisely, the events of the current registration of
653         client.fileHandle are set to client.events and the registration
654         attachment is set to client.
655 
656         If this fails with ENOENT, which means, client.fileHandle turned
657         out not to be registered, a new registration of the client is created
658         using EPOLL_CTL_ADD. This fallback is intended only to be used when a
659         file descriptor is unexpectedly unregistered as it happens when it is
660         closed.
661 
662         Params:
663             client = client to set the conduit registration to
664 
665        Returns:
666             true if everything worked as expected or false if
667             client.fileHandle turned out not to be registered so that
668             a new registration was added.
669 
670         Throws:
671             EpollException on error.
672 
673      **************************************************************************/
674 
675     public bool modify ( ISelectClient client )
676     {
677         if (!this.epollCtl(epoll.CtlOp.EPOLL_CTL_MOD, client.fileHandle,
678             client.events, client))
679         {
680             return false;
681         }
682         else
683         {
684             int errnum = .errno;
685 
686             if (errnum == ENOENT)
687             {
688                 if (!this.epollCtl(epoll.CtlOp.EPOLL_CTL_ADD, client.fileHandle,
689                     client.events, client))
690                 {
691                     return true;
692                 }
693                 else
694                 {
695                     throw this.e.useGlobalErrno().addMessage(
696                         "error adding epoll registration "
697                             ~ "after modification resulted in ENOENT"
698                     );
699                 }
700             }
701             else
702             {
703                 throw this.e.useGlobalErrno().addMessage(
704                     "error modifying epoll registration");
705             }
706         }
707     }
708 
709     /**************************************************************************
710 
711         Sets a timeout manager expiry registration to client if the timeout
712         feature is enabled. This must be done exactly once for each select
713         client that should be able to time out.
714         If the timeout feature is disabled, nothing is done.
715 
716         Params:
717             client = client to set timeout manager expiry registration
718 
719         Returns:
720             true on success or false if the timeout feature is disabled.
721 
722      **************************************************************************/
723 
724     public bool setExpiryRegistration ( ISelectClient client )
725     {
726         if (this._timeout_enabled)
727         {
728             client.expiry_registration = this.timeout_manager.getRegistration(client);
729             return true;
730         }
731         else
732         {
733             return false;
734         }
735     }
736 
737     /***************************************************************************
738 
739         Causes the event loop to exit before entering the next wait cycle.
740 
741         Note that after calling shutdown() the select dispatcher is left in an
742         invalid state, where register() and unregister() calls will cause seg
743         faults and further calls to eventLoop() will exit immediately.
744         (See TODO in eventLoop().)
745 
746      **************************************************************************/
747 
748     public void shutdown ( )
749     {
750         this.register(this.shutdown_event);
751         this.shutdown_event.trigger();
752     }
753 
754     /***************************************************************************
755 
756         While there are clients registered, repeatedly waits for registered
757         events to happen, invokes the corresponding event handlers of the
758         registered clients and unregisters the clients if they desire so.
759 
760         Params:
761             select_cycle_hook = if not null, will be called each time select
762                 cycle finished before waiting for more events. Also called
763                 once before the first select. If returns `true`, epoll will
764                 return immediately if there are no active events.
765             unhandled_exception_hook = if not null, will be called each time
766                 select cycle results in unhandled exception. May either rethrow
767                 or consume (ignore) exception instance after processing it.
768 
769      **************************************************************************/
770 
771     public void eventLoop ( scope bool delegate ( ) select_cycle_hook = null,
772         scope bool delegate (Exception) unhandled_exception_hook  = null )
773     {
774         verify(!this.in_event_loop, "Event loop has already been started.");
775 
776         this.in_event_loop = true;
777         scope ( exit ) this.in_event_loop = false;
778 
779         this.unhandled_exception_hook = unhandled_exception_hook;
780 
781         bool caller_work_pending = false;
782 
783         if (select_cycle_hook !is null)
784             caller_work_pending = select_cycle_hook();
785 
786         while ( (this.registered_clients.length ||
787             this.select_cycle_callbacks.length) &&
788             !this.shutdown_triggered )
789         {
790             caller_work_pending = false;
791 
792             try
793             {
794                 this.select(caller_work_pending ||
795                     this.select_cycle_callbacks.length > 0);
796             }
797             catch (Exception e)
798             {
799                 if (  (unhandled_exception_hook is null)
800                     || !unhandled_exception_hook(e))
801                 {
802                     throw e;
803                 }
804             }
805 
806             if (select_cycle_hook !is null)
807                 caller_work_pending = select_cycle_hook();
808 
809             auto count = this.select_cycle_callbacks.length();
810 
811             for (int i; i < count; i++)
812             {
813                 auto pcb = this.select_cycle_callbacks.pop();
814                 (*pcb)();
815             }
816 
817             this.select_cycle_callbacks.shrink();
818         }
819     }
820 
821     /***************************************************************************
822 
823         Executes an epoll select.
824 
825         Params:
826             exit_asap = if set to 'true', epoll will exit immediately
827                 if there are no events to trigger. Otherwise epoll will wait
828                 indefinitely until any event fires.
829 
830         Returns:
831             the number of epoll keys for which an event was reported.
832 
833      **************************************************************************/
834 
835     protected uint select ( bool exit_asap )
836     {
837         debug ( ISelectClient )
838         {
839             Stderr.formatln("{}.select ({} clients registered):",
840                 typeof(this).stringof, this.registered_clients.length).flush();
841             size_t i;
842             foreach ( client; cast(ClientSet)this.registered_clients )
843             {
844                 Stderr.formatln("   {,3}: {}", i++, client).flush();
845             }
846         }
847 
848         while (true /* actually while epoll_wait is interrupted by a signal */)
849         {
850             ulong us_left = (this.timeout_manager !is null)
851                             ? timeout_manager.us_left
852                             : ulong.max;
853 
854             // Note that timeout_manager.us_left can be ulong.max, too.
855 
856             bool have_timeout = us_left < us_left.max;
857 
858             // have_timeout is true if a timeout is specified, no matter if
859             // epoll_wait actually timed out or not (this is indicated by
860             // n == 0).
861 
862             int epoll_wait_time;
863 
864             if (exit_asap)
865                 epoll_wait_time = 0;
866             else
867             {
868                 if (have_timeout)
869                     epoll_wait_time = cast (int) this.usToMs(us_left);
870                 else
871                     epoll_wait_time = -1;
872             }
873 
874             int n = this.epoll.wait(this.events, epoll_wait_time);
875 
876             version ( EpollCounters ) this.counters.selects++;
877 
878             if (n >= 0)
879             {
880                 debug ( ISelectClient ) if ( !n )
881                 {
882                     Stderr.formatln("{}.select: timed out after {}microsec",
883                             typeof(this).stringof, us_left).flush();
884                 }
885 
886                 version ( EpollCounters ) if ( n == 0 ) this.counters.timeouts++;
887 
888                 this.selected_set = this.events[0 .. n];
889                 scope (exit)
890                     this.selected_set = null;
891 
892                 this.handle(this.selected_set, this.unhandled_exception_hook);
893 
894                 return n;
895             }
896             else
897             {
898                 int errnum = .errno;
899 
900                 if (errnum != EINTR)
901                 {
902                     throw this.e.useGlobalErrno().addMessage(
903                         "error waiting for epoll events");
904                 }
905             }
906         }
907     }
908 
909     /***************************************************************************
910 
911         Creates/deletes/modifies registration inside the epoll. Wrapped to
912         allow for the sanity checks on the EpollSelectDispatcher's level.
913 
914         Params:
915             op     = epoll_ctl opcode
916             fd     = file descriptor to register for events
917             events = events to register fd for
918             client = user ISelectClient to set data.obj of the created epoll_data_t
919                      instance to
920 
921         Returns:
922             0 on success or -1 on error. On error errno is set appropriately.
923 
924     ***************************************************************************/
925 
926     private int epollCtl ( Epoll.CtlOp op, int fd, Epoll.Event events,
927             ISelectClient client )
928     {
929         debug (EpollFdSanity)
930         {
931             return this.epoll.ctl(op, fd, client.events,
932                     FdObjEpollData.encode(client, client.fileHandle));
933         }
934         else
935         {
936             return this.epoll.ctl(op, fd, client.events, client);
937         }
938     }
939 
940     /***************************************************************************
941 
942         Called when the shutdown event fires (via a call to the shutdown()
943         method). Sets the shutdown flag, ensuring that the event loop will exit,
944         regardless of whether there are any clients still registered.
945 
946         Returns:
947             true to stay registered in the selector
948 
949      **************************************************************************/
950 
951     private bool shutdownTrigger ( )
952     {
953         this.shutdown_triggered = true;
954 
955         return true;
956     }
957 
958     /***************************************************************************
959 
960         Converts a microseconds value to milliseconds for use in select().
961         It is crucial that this conversion always rounds up. Otherwise the
962         timeout manager might not find a timed out client after select() has
963         reported a timeout.
964 
965         Params:
966             us = time value in microseconds
967 
968         Returns:
969             nearest time value in milliseconds that is not less than us.
970 
971      **************************************************************************/
972 
973     private static ulong usToMs ( ulong us )
974     {
975         ulong ms = us / 1000;
976 
977         return ms + ((us - ms * 1000) != 0);
978     }
979 }