// SPDX-License-Identifier: Apache-2.0 OR MIT

// -----------------------------------------------------------------------------
// Lock-free implementations

#[cfg(not(any(
    all(
        portable_atomic_no_atomic_load_store,
        not(all(target_arch = "bpf", not(feature = "critical-section"))),
    ),
    portable_atomic_unsafe_assume_single_core,
    target_arch = "avr",
    target_arch = "msp430",
)))]
#[cfg_attr(
    portable_atomic_no_cfg_target_has_atomic,
    cfg(not(all(
        any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"),
        portable_atomic_no_atomic_cas,
    )))
)]
#[cfg_attr(
    not(portable_atomic_no_cfg_target_has_atomic),
    cfg(not(all(
        any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"),
        not(target_has_atomic = "ptr"),
    )))
)]
mod core_atomic;

// aarch64 128-bit atomics
#[cfg(all(
    target_arch = "aarch64",
    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(
    all(any(miri, portable_atomic_sanitize_thread), portable_atomic_new_atomic_intrinsics),
    path = "atomic128/intrinsics.rs"
)]
#[cfg_attr(
    not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_new_atomic_intrinsics)),
    path = "atomic128/aarch64.rs"
)]
mod aarch64;

// x86_64 128-bit atomics
#[cfg(all(
    target_arch = "x86_64",
    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
    any(
        target_feature = "cmpxchg16b",
        portable_atomic_target_feature = "cmpxchg16b",
        all(
            feature = "fallback",
            not(portable_atomic_no_cmpxchg16b_target_feature),
            not(portable_atomic_no_outline_atomics),
            not(any(target_env = "sgx", miri)),
        ),
    ),
))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "atomic128/intrinsics.rs")]
#[cfg_attr(not(any(miri, portable_atomic_sanitize_thread)), path = "atomic128/x86_64.rs")]
mod x86_64;

// powerpc64 128-bit atomics
#[cfg(all(
    target_arch = "powerpc64",
    portable_atomic_unstable_asm_experimental_arch,
    any(
        target_feature = "quadword-atomics",
        portable_atomic_target_feature = "quadword-atomics",
        all(
            feature = "fallback",
            not(portable_atomic_no_outline_atomics),
            any(test, portable_atomic_outline_atomics), // TODO(powerpc64): currently disabled by default
            any(
                all(
                    target_os = "linux",
                    any(
                        target_env = "gnu",
                        all(
                            any(target_env = "musl", target_env = "ohos"),
                            not(target_feature = "crt-static"),
                        ),
                        portable_atomic_outline_atomics,
                    ),
                ),
                target_os = "android",
                target_os = "freebsd",
            ),
            not(any(miri, portable_atomic_sanitize_thread)),
        ),
    ),
))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(
    all(any(miri, portable_atomic_sanitize_thread), portable_atomic_llvm_15),
    path = "atomic128/intrinsics.rs"
)]
#[cfg_attr(
    not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_llvm_15)),
    path = "atomic128/powerpc64.rs"
)]
mod powerpc64;

// s390x 128-bit atomics
#[cfg(all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch))]
// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "atomic128/intrinsics.rs")]
#[cfg_attr(not(any(miri, portable_atomic_sanitize_thread)), path = "atomic128/s390x.rs")]
mod s390x;

// pre-v6 ARM Linux 64-bit atomics
#[cfg(feature = "fallback")]
// Miri and Sanitizer do not support inline assembly.
#[cfg(all(
    target_arch = "arm",
    not(any(miri, portable_atomic_sanitize_thread)),
    not(portable_atomic_no_asm),
    any(target_os = "linux", target_os = "android"),
    not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
    not(portable_atomic_no_outline_atomics),
))]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
mod arm_linux;

// MSP430 atomics
#[cfg(target_arch = "msp430")]
pub(crate) mod msp430;

// atomic load/store for RISC-V without A-extension
#[cfg(any(test, not(feature = "critical-section")))]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
#[cfg_attr(
    not(portable_atomic_no_cfg_target_has_atomic),
    cfg(any(test, not(target_has_atomic = "ptr")))
)]
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
mod riscv;

// x86-specific optimizations
// Miri and Sanitizer do not support inline assembly.
#[cfg(all(
    any(target_arch = "x86", target_arch = "x86_64"),
    not(any(miri, portable_atomic_sanitize_thread)),
    not(portable_atomic_no_asm),
))]
mod x86;

// -----------------------------------------------------------------------------
// Lock-based fallback implementations

#[cfg(feature = "fallback")]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
#[cfg(any(
    test,
    not(any(
        all(
            target_arch = "aarch64",
            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
        ),
        all(
            target_arch = "x86_64",
            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
            any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
        ),
        all(
            target_arch = "powerpc64",
            portable_atomic_unstable_asm_experimental_arch,
            any(
                target_feature = "quadword-atomics",
                portable_atomic_target_feature = "quadword-atomics",
            ),
        ),
        all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch),
    ))
))]
mod fallback;

// -----------------------------------------------------------------------------
// Critical section based fallback implementations

// On AVR, we always use critical section based fallback implementation.
// AVR can be safely assumed to be single-core, so this is sound.
// https://github.com/llvm/llvm-project/blob/llvmorg-17.0.0-rc2/llvm/lib/Target/AVR/AVRExpandPseudoInsts.cpp#L1074
// MSP430 as well.
#[cfg(any(
    all(test, target_os = "none"),
    portable_atomic_unsafe_assume_single_core,
    feature = "critical-section",
    target_arch = "avr",
    target_arch = "msp430",
))]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
#[cfg_attr(
    not(portable_atomic_no_cfg_target_has_atomic),
    cfg(any(test, not(target_has_atomic = "ptr")))
)]
#[cfg(any(
    target_arch = "arm",
    target_arch = "avr",
    target_arch = "msp430",
    target_arch = "riscv32",
    target_arch = "riscv64",
    target_arch = "xtensa",
    feature = "critical-section",
))]
mod interrupt;

// -----------------------------------------------------------------------------
// Atomic float implementations

#[cfg(feature = "float")]
pub(crate) mod float;

// -----------------------------------------------------------------------------

#[cfg(not(any(
    portable_atomic_no_atomic_load_store,
    portable_atomic_unsafe_assume_single_core,
    target_arch = "avr",
    target_arch = "msp430",
)))]
#[cfg_attr(
    portable_atomic_no_cfg_target_has_atomic,
    cfg(not(all(
        any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"),
        portable_atomic_no_atomic_cas,
    )))
)]
#[cfg_attr(
    not(portable_atomic_no_cfg_target_has_atomic),
    cfg(not(all(
        any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"),
        not(target_has_atomic = "ptr"),
    )))
)]
items! {
    pub(crate) use self::core_atomic::{
        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
        AtomicUsize,
    };
    #[cfg_attr(
        portable_atomic_no_cfg_target_has_atomic,
        cfg(any(
            not(portable_atomic_no_atomic_64),
            not(any(target_pointer_width = "16", target_pointer_width = "32")),
        ))
    )]
    #[cfg_attr(
        not(portable_atomic_no_cfg_target_has_atomic),
        cfg(any(
            target_has_atomic = "64",
            not(any(target_pointer_width = "16", target_pointer_width = "32")),
        ))
    )]
    pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
}
// bpf
#[cfg(all(
    target_arch = "bpf",
    portable_atomic_no_atomic_load_store,
    not(feature = "critical-section"),
))]
pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize};

// RISC-V without A-extension & !(assume single core | critical section)
#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
items! {
    pub(crate) use self::riscv::{
        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
        AtomicUsize,
    };
    #[cfg(target_arch = "riscv64")]
    pub(crate) use self::riscv::{AtomicI64, AtomicU64};
}

// no core atomic CAS & (assume single core | critical section) => critical section based fallback
#[cfg(any(
    portable_atomic_unsafe_assume_single_core,
    feature = "critical-section",
    target_arch = "avr",
    target_arch = "msp430",
))]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
items! {
    pub(crate) use self::interrupt::{
        AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize,
    };
    #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
    pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
    #[cfg(any(
        not(any(target_pointer_width = "16", target_pointer_width = "32")),
        feature = "fallback",
    ))]
    pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
    #[cfg(feature = "fallback")]
    pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
}

// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback
#[cfg(feature = "fallback")]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
items! {
    #[cfg(not(all(
        target_arch = "arm",
        not(any(miri, portable_atomic_sanitize_thread)),
        not(portable_atomic_no_asm),
        any(target_os = "linux", target_os = "android"),
        not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
        not(portable_atomic_no_outline_atomics),
    )))]
    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
    pub(crate) use self::fallback::{AtomicI64, AtomicU64};
    #[cfg(not(any(
        all(
            target_arch = "aarch64",
            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
        ),
        all(
            target_arch = "x86_64",
            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
            any(
                target_feature = "cmpxchg16b",
                portable_atomic_target_feature = "cmpxchg16b",
                all(
                    feature = "fallback",
                    not(portable_atomic_no_cmpxchg16b_target_feature),
                    not(portable_atomic_no_outline_atomics),
                    not(any(target_env = "sgx", miri)),
                ),
            ),
        ),
        all(
            target_arch = "powerpc64",
            portable_atomic_unstable_asm_experimental_arch,
            any(
                target_feature = "quadword-atomics",
                portable_atomic_target_feature = "quadword-atomics",
                all(
                    feature = "fallback",
                    not(portable_atomic_no_outline_atomics),
                    portable_atomic_outline_atomics, // TODO(powerpc64): currently disabled by default
                    any(
                        all(
                            target_os = "linux",
                            any(
                                target_env = "gnu",
                                all(
                                    any(target_env = "musl", target_env = "ohos"),
                                    not(target_feature = "crt-static"),
                                ),
                                portable_atomic_outline_atomics,
                            ),
                        ),
                        target_os = "android",
                        target_os = "freebsd",
                    ),
                    not(any(miri, portable_atomic_sanitize_thread)),
                ),
            ),
        ),
        all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch),
    )))]
    pub(crate) use self::fallback::{AtomicI128, AtomicU128};
}

// 64-bit atomics (platform-specific)
// pre-v6 ARM Linux
#[cfg(feature = "fallback")]
#[cfg(all(
    target_arch = "arm",
    not(any(miri, portable_atomic_sanitize_thread)),
    not(portable_atomic_no_asm),
    any(target_os = "linux", target_os = "android"),
    not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
    not(portable_atomic_no_outline_atomics),
))]
#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
pub(crate) use self::arm_linux::{AtomicI64, AtomicU64};

// 128-bit atomics (platform-specific)
// aarch64
#[cfg(all(
    target_arch = "aarch64",
    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
))]
pub(crate) use self::aarch64::{AtomicI128, AtomicU128};
// x86_64 & (cmpxchg16b | outline-atomics)
#[cfg(all(
    target_arch = "x86_64",
    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
    any(
        target_feature = "cmpxchg16b",
        portable_atomic_target_feature = "cmpxchg16b",
        all(
            feature = "fallback",
            not(portable_atomic_no_cmpxchg16b_target_feature),
            not(portable_atomic_no_outline_atomics),
            not(any(target_env = "sgx", miri)),
        ),
    ),
))]
pub(crate) use self::x86_64::{AtomicI128, AtomicU128};
// powerpc64 & (pwr8 | outline-atomics)
#[cfg(all(
    target_arch = "powerpc64",
    portable_atomic_unstable_asm_experimental_arch,
    any(
        target_feature = "quadword-atomics",
        portable_atomic_target_feature = "quadword-atomics",
        all(
            feature = "fallback",
            not(portable_atomic_no_outline_atomics),
            portable_atomic_outline_atomics, // TODO(powerpc64): currently disabled by default
            any(
                all(
                    target_os = "linux",
                    any(
                        target_env = "gnu",
                        all(
                            any(target_env = "musl", target_env = "ohos"),
                            not(target_feature = "crt-static"),
                        ),
                        portable_atomic_outline_atomics,
                    ),
                ),
                target_os = "android",
                target_os = "freebsd",
            ),
            not(any(miri, portable_atomic_sanitize_thread)),
        ),
    ),
))]
pub(crate) use self::powerpc64::{AtomicI128, AtomicU128};
// s390x
#[cfg(all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch))]
pub(crate) use self::s390x::{AtomicI128, AtomicU128};