1mod call;
9mod call_scope;
10mod event;
11mod registry;
12pub use call::{ForeignCall, ForeignCallback, ForeignCallbackHandle};
13pub use event::ForeignEvent;
14pub(crate) use registry::Registry;
15pub(crate) use registry::Retired;
16
17use super::{ClientId, Inner, RuntimeOwner, SlotDomain};
18use crate::{Error, Result};
19use std::sync::{Arc, Mutex};
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub enum ForeignScope {
25 Module,
27 Slot(u64),
29 Session {
31 slot: u64,
33 handle: u64,
35 },
36}
37impl ForeignScope {
38 pub(super) fn slot(self) -> Option<u64> {
39 match self {
40 Self::Module => None,
41 Self::Slot(slot) | Self::Session { slot, .. } => Some(slot),
42 }
43 }
44}
45
46#[derive(Clone, Copy, Debug, PartialEq, Eq)]
49pub enum ForeignWorkKind {
50 Call,
53 Session,
55 Operation,
60}
61
62#[derive(Clone, Copy, Debug, PartialEq, Eq)]
64pub enum ForeignAccess {
65 Session,
67 Token,
70}
71
72#[derive(Clone, Copy, Debug, PartialEq, Eq)]
75pub enum ForeignStep {
76 Start,
78 Continue,
80 Complete,
82 Cancel,
84 GetId,
86 Join,
88 CloseSession,
90}
91
92impl ForeignStep {
93 fn controls_parent(self, pending_parent: bool) -> bool {
94 matches!(self, Self::Complete | Self::GetId | Self::Join)
95 || self == Self::Cancel && pending_parent
96 }
97
98 fn memory_returned(self, code: u64, pending_parent: bool) -> bool {
99 if self.controls_parent(pending_parent) {
100 crate::native_return::async_memory_returned(code)
101 } else {
102 crate::native_return::diagnostic_memory_returned(code)
103 }
104 }
105
106 fn context_scope(self, code: u64, pending_parent: bool) -> crate::native_return::FailureScope {
107 if !self.memory_returned(code, pending_parent) {
108 crate::native_return::FailureScope::Module
109 } else {
110 crate::native_return::diagnostic_context_scope(code)
111 }
112 }
113}
114
115#[derive(Clone, Copy, Debug, PartialEq, Eq)]
119pub enum ForeignCompletion {
120 Returned,
122 Pending,
125 Continue,
128 Detached,
131 Uncertain,
134}
135
136#[derive(Clone, Copy, Debug, PartialEq, Eq)]
139pub enum ForeignProgress {
140 Pending,
142 Sequence,
144}
145
146pub struct ForeignReceipt {
149 pub code: u64,
151 pub publication: Result<()>,
153}
154
155#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
158pub struct ForeignWorkId(u64);
159
160#[derive(Clone, Copy)]
161struct NativeReturn {
162 step: ForeignStep,
163 code: u64,
164}
165
166#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
169pub struct ForeignStatus {
170 pub participants: usize,
172 pub sessions: usize,
174 pub progressable: usize,
176 pub uncertain: usize,
178 pub callbacks: usize,
180}
181
182#[derive(Clone, Copy, PartialEq, Eq)]
183enum Progress {
184 Sequence,
185 Pending,
186}
187
188#[derive(Clone, Copy, PartialEq, Eq)]
189enum Phase {
190 Prepared,
191 PreparedJoin,
192 Reserved(ForeignStep),
193 Issued(ForeignStep),
194 Progressable(Progress),
195 SessionOpen,
196 Detached,
197 JoinReserved,
198 Returned,
199 Uncertain,
200}
201
202struct Participant {
203 id: ClientId,
204 image: Arc<crate::native::image::LoadedImage>,
205 owner: Mutex<Option<Box<dyn Send + Sync>>>,
208 finalized: std::sync::atomic::AtomicBool,
209 callbacks: std::sync::atomic::AtomicUsize,
210 untracked: Arc<std::sync::atomic::AtomicBool>,
211}
212
213impl Drop for Participant {
214 fn drop(&mut self) {
215 let owner = self
216 .owner
217 .get_mut()
218 .unwrap_or_else(|e| e.into_inner())
219 .take();
220 if !self.finalized.load(std::sync::atomic::Ordering::Acquire) {
221 if let Some(owner) = owner {
222 std::mem::forget(owner);
223 }
224 } else {
225 drop(owner);
226 }
227 }
228}
229
230struct WorkState {
231 phase: Phase,
232 last_return: Option<NativeReturn>,
233 graph: Option<Box<dyn Send + Sync>>,
234}
235
236struct Work {
237 native_retired: std::sync::atomic::AtomicBool,
238 token_epoch: Option<u64>,
239 id: ForeignWorkId,
240 participant: Arc<Participant>,
241 kind: ForeignWorkKind,
242 scope: ForeignScope,
243 session_handle: std::sync::atomic::AtomicU64,
246 access: ForeignAccess,
247 domain: Option<Arc<SlotDomain>>,
248 state: Mutex<WorkState>,
249 session: Option<Arc<Work>>,
252 join_source: Option<Arc<Work>>,
253 gate: crate::admission_wait::ResourceGate,
254 held: std::sync::atomic::AtomicBool,
255 callbacks: std::sync::atomic::AtomicUsize,
256}
257
258impl ForeignParticipant {
259 pub unsafe fn adopt_operation<T: Send + Sync + 'static>(
275 &self,
276 session: &ForeignWork,
277 kind: ForeignWorkKind,
278 access: ForeignAccess,
279 progress: ForeignProgress,
280 step: ForeignStep,
281 last_return: u64,
282 owner: T,
283 ) -> Result<ForeignWork> {
284 let mut unpublished = UnpublishedGraph {
285 graph: Some(Box::new(owner)),
286 participant: self.participant.clone(),
287 native: true,
288 };
289 if !Arc::ptr_eq(&self.inner, &session.inner)
290 || !Arc::ptr_eq(&self.participant, &session.work.participant)
291 || session.work.kind != ForeignWorkKind::Session
292 || kind == ForeignWorkKind::Session
293 {
294 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
295 crate::ExternalExecutionRejection::ForeignOperationSessionMismatch,
296 )));
297 }
298 let rv = fe11_sys::CK_RV::try_from(last_return).map_err(|_| {
299 Error::Rejected(crate::RejectionReason::ExternalExecution(
300 crate::ExternalExecutionRejection::ForeignReturnCodeConversion,
301 ))
302 })?;
303 let phase = match progress {
304 ForeignProgress::Pending
305 if rv == fe11_sys::CKR_PENDING
306 && kind == ForeignWorkKind::Operation
307 && access == ForeignAccess::Session =>
308 {
309 Progress::Pending
310 }
311 ForeignProgress::Sequence if step.memory_returned(last_return, false) => {
312 Progress::Sequence
313 }
314 _ => {
315 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
316 crate::ExternalExecutionRejection::ForeignAdoptedProgressMismatch,
317 )))
318 }
319 };
320 let work = self.register_work(
324 session.work.scope(),
325 kind,
326 access,
327 Some(session.work.clone()),
328 Phase::Progressable(phase),
329 unpublished.graph.take().expect("adopted graph"),
330 None,
331 call::AdmissionMode::Ordinary,
332 )?;
333 work.work
334 .state
335 .lock()
336 .map_err(|_| Error::Poisoned)?
337 .last_return = Some(NativeReturn {
338 step,
339 code: last_return,
340 });
341 Ok(work)
342 }
343
344 pub unsafe fn prepare_session_open<T: Send + Sync + 'static>(
352 &self,
353 slot: u64,
354 owner: T,
355 ) -> Result<ForeignWork> {
356 self.register_work(
357 ForeignScope::Slot(slot),
358 ForeignWorkKind::Session,
359 ForeignAccess::Token,
360 None,
361 Phase::Prepared,
362 Box::new(owner),
363 None,
364 call::AdmissionMode::Ordinary,
365 )
366 }
367
368 pub unsafe fn adopt_session<T: Send + Sync + 'static>(
378 &self,
379 slot: u64,
380 handle: u64,
381 owner: T,
382 ) -> Result<ForeignWork> {
383 self.register_work(
384 ForeignScope::Session { slot, handle },
385 ForeignWorkKind::Session,
386 ForeignAccess::Token,
387 None,
388 Phase::SessionOpen,
389 Box::new(owner),
390 None,
391 call::AdmissionMode::Ordinary,
392 )
393 }
394
395 pub unsafe fn prepare_call<T: Send + Sync + 'static>(
406 &self,
407 scope: ForeignScope,
408 owner: T,
409 ) -> Result<ForeignWork> {
410 if matches!(scope, ForeignScope::Session { .. }) {
411 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
412 crate::ExternalExecutionRejection::ForeignSessionOwnerRequired,
413 )));
414 }
415 self.register_work(
416 scope,
417 ForeignWorkKind::Call,
418 ForeignAccess::Token,
419 None,
420 Phase::Prepared,
421 Box::new(owner),
422 None,
423 call::AdmissionMode::Ordinary,
424 )
425 }
426
427 pub unsafe fn prepare_operation<T: Send + Sync + 'static>(
440 &self,
441 session: &ForeignWork,
442 kind: ForeignWorkKind,
443 access: ForeignAccess,
444 owner: T,
445 ) -> Result<ForeignWork> {
446 if !Arc::ptr_eq(&self.inner, &session.inner)
447 || !Arc::ptr_eq(&self.participant, &session.work.participant)
448 || session.work.kind != ForeignWorkKind::Session
449 || kind == ForeignWorkKind::Session
450 {
451 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
452 crate::ExternalExecutionRejection::ForeignOperationSessionMismatch,
453 )));
454 }
455 self.register_work(
456 session.work.scope(),
457 kind,
458 access,
459 Some(session.work.clone()),
460 Phase::Prepared,
461 Box::new(owner),
462 None,
463 call::AdmissionMode::Ordinary,
464 )
465 }
466
467 pub unsafe fn prepare_join<T: Send + Sync + 'static>(
476 &self,
477 source: &ForeignWork,
478 session: &ForeignWork,
479 owner: T,
480 ) -> Result<ForeignWork> {
481 self.prepare_join_admitted(
482 source,
483 session,
484 Box::new(owner),
485 call::AdmissionMode::Ordinary,
486 )
487 }
488
489 fn prepare_join_admitted(
490 &self,
491 source: &ForeignWork,
492 session: &ForeignWork,
493 owner: Box<dyn Send + Sync>,
494 mode: call::AdmissionMode,
495 ) -> Result<ForeignWork> {
496 if !Arc::ptr_eq(&self.inner, &source.inner)
497 || !Arc::ptr_eq(&self.inner, &session.inner)
498 || !Arc::ptr_eq(&self.participant, &source.work.participant)
499 || !Arc::ptr_eq(&self.participant, &session.work.participant)
500 || session.work.kind != ForeignWorkKind::Session
501 {
502 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
503 crate::ExternalExecutionRejection::ForeignJoinOwnerMismatch,
504 )));
505 }
506 if source.work.state.lock().map_err(|_| Error::Poisoned)?.phase != Phase::Detached {
507 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
508 crate::ExternalExecutionRejection::ForeignJoinSourceNotDetached,
509 )));
510 }
511 self.register_work(
512 session.work.scope(),
513 ForeignWorkKind::Operation,
514 ForeignAccess::Session,
515 Some(session.work.clone()),
516 Phase::PreparedJoin,
517 owner,
518 Some(source.work.clone()),
519 mode,
520 )
521 }
522
523 fn register_work(
524 &self,
525 scope: ForeignScope,
526 kind: ForeignWorkKind,
527 access: ForeignAccess,
528 session: Option<Arc<Work>>,
529 phase: Phase,
530 graph: Box<dyn Send + Sync>,
531 join_source: Option<Arc<Work>>,
532 mode: call::AdmissionMode,
533 ) -> Result<ForeignWork> {
534 let mut unpublished = UnpublishedGraph {
537 graph: Some(graph),
538 participant: self.participant.clone(),
539 native: matches!(phase, Phase::SessionOpen | Phase::Progressable(_)),
540 };
541 self.inner.check_lifecycle_reentry()?;
542 if let ForeignScope::Session { handle, .. } = scope {
543 let handle = fe11_sys::CK_SESSION_HANDLE::try_from(handle).map_err(|_| {
544 Error::Rejected(crate::RejectionReason::ExternalExecution(
545 crate::ExternalExecutionRejection::ForeignSessionHandleConversion,
546 ))
547 })?;
548 if handle == fe11_sys::CK_INVALID_HANDLE {
549 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
550 crate::ExternalExecutionRejection::ForeignSessionHandleInvalid,
551 )));
552 }
553 }
554 let domain = scope
555 .slot()
556 .map(|slot| self.inner.foreign.domain(slot))
557 .transpose()?;
558 let work = Arc::new(Work {
559 native_retired: std::sync::atomic::AtomicBool::new(false),
560 token_epoch: domain.as_ref().map(|domain| {
561 domain
562 .foreign_epoch
563 .load(std::sync::atomic::Ordering::Acquire)
564 }),
565 id: ForeignWorkId(ClientId::allocate(&self.inner.next_client)?.value()),
566 participant: self.participant.clone(),
567 kind,
568 scope,
569 access,
570 domain,
571 session_handle: std::sync::atomic::AtomicU64::new(match scope {
572 ForeignScope::Session { handle, .. } if kind == ForeignWorkKind::Session => handle,
573 _ => 0,
574 }),
575 state: Mutex::new(WorkState {
576 phase,
577 last_return: None,
578 graph: unpublished.graph.take(),
579 }),
580 session,
581 join_source,
582 gate: crate::admission_wait::ResourceGate::new(crate::BusyReason::ExternalExecution(
583 crate::ExternalExecutionBusy::ForeignSessionActivity,
584 )),
585 held: std::sync::atomic::AtomicBool::new(true),
586 callbacks: std::sync::atomic::AtomicUsize::new(0),
587 });
588 let admission = call::admit(
589 &self.inner,
590 work.domain.as_deref(),
591 None,
592 ForeignAccess::Session,
593 mode,
594 )?;
595 self.inner.foreign.check_session(&work)?;
596 self.inner.foreign.publish(&work)?;
597 drop(admission);
598 Ok(ForeignWork {
599 inner: self.inner.clone(),
600 work,
601 })
602 }
603
604 pub fn take_abandoned_work(&self) -> Result<Option<ForeignWork>> {
607 self.inner.foreign.abandoned(&self.participant).map(|work| {
608 work.map(|work| ForeignWork {
609 inner: self.inner.clone(),
610 work,
611 })
612 })
613 }
614}
615
616impl ForeignWork {
617 pub fn id(&self) -> ForeignWorkId {
620 self.work.id
621 }
622
623 pub fn last_native_step(&self) -> Result<Option<ForeignStep>> {
625 Ok(self
626 .work
627 .state
628 .lock()
629 .map_err(|_| Error::Poisoned)?
630 .last_return
631 .map(|value| value.step))
632 }
633
634 pub fn scope(&self) -> ForeignScope {
636 self.work.scope()
637 }
638
639 pub fn last_native_return(&self) -> Result<Option<u64>> {
641 Ok(self
642 .work
643 .state
644 .lock()
645 .map_err(|_| Error::Poisoned)?
646 .last_return
647 .map(|value| value.code))
648 }
649}
650
651impl Work {
652 fn scope(&self) -> ForeignScope {
656 let handle = self
657 .session_handle
658 .load(std::sync::atomic::Ordering::Acquire);
659 if self.kind == ForeignWorkKind::Session && handle != 0 {
660 ForeignScope::Session {
661 slot: self.scope.slot().expect("session slot"),
662 handle,
663 }
664 } else {
665 self.scope
666 }
667 }
668}
669
670struct UnpublishedGraph {
671 graph: Option<Box<dyn Send + Sync>>,
672 participant: Arc<Participant>,
673 native: bool,
674}
675impl Drop for UnpublishedGraph {
676 fn drop(&mut self) {
677 if self.native {
678 if let Some(graph) = self.graph.take() {
679 self.participant
680 .untracked
681 .store(true, std::sync::atomic::Ordering::Release);
682 std::mem::forget(graph);
683 std::mem::forget(self.participant.clone());
684 }
685 }
686 }
687}
688
689impl Drop for Work {
690 fn drop(&mut self) {
691 let state = self.state.get_mut().unwrap_or_else(|e| e.into_inner());
692 let graph = state.graph.take();
693 if !matches!(
694 state.phase,
695 Phase::Prepared
696 | Phase::PreparedJoin
697 | Phase::Returned
698 | Phase::Detached
699 | Phase::JoinReserved
700 ) {
701 std::mem::forget(self.participant.clone());
704 self.participant
705 .untracked
706 .store(true, std::sync::atomic::Ordering::Release);
707 if let Some(graph) = graph {
708 std::mem::forget(graph);
709 }
710 } else {
711 drop(graph);
712 }
713 }
714}
715
716pub struct ForeignParticipant {
719 inner: Arc<Inner>,
720 participant: Arc<Participant>,
721}
722
723pub struct ForeignWork {
726 inner: Arc<Inner>,
727 work: Arc<Work>,
728}
729
730impl Drop for ForeignWork {
731 fn drop(&mut self) {
732 self.work
733 .held
734 .store(false, std::sync::atomic::Ordering::Release);
735 self.inner.context_wakes.notify();
736 }
737}
738
739impl RuntimeOwner {
740 pub unsafe fn register_foreign<T: Send + Sync + 'static>(
755 &self,
756 owner: T,
757 ) -> Result<ForeignParticipant> {
758 self.module()
759 .inner
760 .foreign
761 .register(&self.module().inner, Box::new(owner))
762 }
763
764 pub fn foreign_status(&self) -> Result<ForeignStatus> {
766 self.module().inner.foreign.status()
767 }
768
769 pub fn collect_foreign(&self) -> Result<usize> {
774 self.module().inner.check_lifecycle_reentry()?;
775 self.module().inner.foreign.collect_completed()
776 }
777}
778
779impl Inner {
780 pub(crate) fn check_lifecycle_reentry(&self) -> Result<()> {
781 let image = self.foreign.image_id();
782 if crate::notify::in_callback_for(image) || call::in_call(image) {
783 return Err(Error::Busy(crate::BusyReason::ExternalExecution(
784 crate::ExternalExecutionBusy::NativeLifecycleReentry,
785 )));
786 }
787 Ok(())
788 }
789}
790
791impl super::ModuleDrain {
792 pub unsafe fn prepare_foreign_join<T: Send + Sync + 'static>(
797 &self,
798 participant: &ForeignParticipant,
799 source: &ForeignWork,
800 session: &ForeignWork,
801 owner: T,
802 ) -> Result<ForeignWork> {
803 if !Arc::ptr_eq(&self.inner, &participant.inner) {
804 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
805 crate::ExternalExecutionRejection::ForeignDrainingRuntime,
806 )));
807 }
808 participant.prepare_join_admitted(
809 source,
810 session,
811 Box::new(owner),
812 call::AdmissionMode::Module,
813 )
814 }
815 pub fn begin_foreign<'a>(
817 &'a self,
818 work: &'a mut ForeignWork,
819 step: ForeignStep,
820 ) -> Result<ForeignCall<'a>> {
821 if !Arc::ptr_eq(&self.inner, &work.inner) {
822 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
823 crate::ExternalExecutionRejection::ForeignDrainingRuntime,
824 )));
825 }
826 work.begin_admitted(step, call::AdmissionMode::Module)
827 }
828}
829
830impl super::TokenDrain {
831 pub unsafe fn prepare_foreign_join<T: Send + Sync + 'static>(
836 &self,
837 participant: &ForeignParticipant,
838 source: &ForeignWork,
839 session: &ForeignWork,
840 owner: T,
841 ) -> Result<ForeignWork> {
842 if !Arc::ptr_eq(&self.context.inner, &participant.inner)
843 || !session
844 .work
845 .domain
846 .as_ref()
847 .is_some_and(|domain| Arc::ptr_eq(domain, &self.context.snapshot.domain))
848 {
849 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
850 crate::ExternalExecutionRejection::ForeignDrainingSlot,
851 )));
852 }
853 participant.prepare_join_admitted(
854 source,
855 session,
856 Box::new(owner),
857 call::AdmissionMode::Token,
858 )
859 }
860 pub fn begin_foreign<'a>(
862 &'a self,
863 work: &'a mut ForeignWork,
864 step: ForeignStep,
865 ) -> Result<ForeignCall<'a>> {
866 if !Arc::ptr_eq(&self.context.inner, &work.inner)
867 || !work
868 .work
869 .domain
870 .as_ref()
871 .is_some_and(|domain| Arc::ptr_eq(domain, &self.context.snapshot.domain))
872 {
873 return Err(Error::Rejected(crate::RejectionReason::ExternalExecution(
874 crate::ExternalExecutionRejection::ForeignDrainingSlot,
875 )));
876 }
877 work.begin_admitted(step, call::AdmissionMode::Token)
878 }
879}