Commit 96d7852c authored by Andy Polyakov's avatar Andy Polyakov
Browse files

internal/refcount.h: overhaul fencing and add _MSC_VER section. 



Relax memory_order on counter decrement itself, because mutable
members of the reference-counted structure should be visible on all
processors independently on counter. [Even re-format and minimize
dependency on other headers.]

Reviewed-by: default avatarKurt Roeckx <kurt@roeckx.be>
(Merged from https://github.com/openssl/openssl/pull/6900)
parent 86ed2e1c

include/internal/refcount.h

+62 −11
Original line number Diff line number Diff line 
/*

 * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.

 * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.

 *

 * Licensed under the OpenSSL license (the "License").  You may not use

 * this file except in compliance with the License.  You can obtain a copy
@@ -29,15 +29,25 @@ 


typedef _Atomic int CRYPTO_REF_COUNT;



static ossl_inline int CRYPTO_UP_REF(_Atomic int *val, int *ret, void *lock)

static inline int CRYPTO_UP_REF(_Atomic int *val, int *ret, void *lock)

{

    *ret = atomic_fetch_add_explicit(val, 1, memory_order_relaxed) + 1;

    return 1;

}



static ossl_inline int CRYPTO_DOWN_REF(_Atomic int *val, int *ret, void *lock)

/*

 * Changes to shared structure other than reference counter have to be

 * serialized. And any kind of serialization implies a release fence. This

 * means that by the time reference counter is decremented all other

 * changes are visible on all processors. Hence decrement itself can be

 * relaxed. In case it hits zero, object will be destructed. Since it's

 * last use of the object, destructor programmer might reason that access

 * to mutable members doesn't have to be serialized anymore, which would

 * otherwise imply an acquire fence. Hence conditional acquire fence...

 */

static inline int CRYPTO_DOWN_REF(_Atomic int *val, int *ret, void *lock)

{

    *ret = atomic_fetch_sub_explicit(val, 1, memory_order_release) - 1;

    *ret = atomic_fetch_sub_explicit(val, 1, memory_order_relaxed) - 1;

    if (*ret == 0)

        atomic_thread_fence(memory_order_acquire);

    return 1;
@@ -49,20 +59,61 @@ static ossl_inline int CRYPTO_DOWN_REF(_Atomic int *val, int *ret, void *lock) 


typedef int CRYPTO_REF_COUNT;



static ossl_inline int CRYPTO_UP_REF(int *val, int *ret, void *lock)

static __inline__ int CRYPTO_UP_REF(int *val, int *ret, void *lock)

{

    *ret = __atomic_fetch_add(val, 1, __ATOMIC_RELAXED) + 1;

    return 1;

}



static ossl_inline int CRYPTO_DOWN_REF(int *val, int *ret, void *lock)

static __inline__ int CRYPTO_DOWN_REF(int *val, int *ret, void *lock)

{

    *ret = __atomic_fetch_sub(val, 1, __ATOMIC_RELEASE) - 1;

    *ret = __atomic_fetch_sub(val, 1, __ATOMIC_RELAXED) - 1;

    if (*ret == 0)

        __atomic_thread_fence(__ATOMIC_ACQUIRE);

    return 1;

}



# elif defined(_MSC_VER) && _MSC_VER>=1200



#  define HAVE_ATOMICS 1



typedef volatile int CRYPTO_REF_COUNT;



#  if (defined(_M_ARM) && _M_ARM>=7) || defined(_M_ARM64)

#   include <intrin.h>

#   if defined(_M_ARM64) && !defined(_ARM_BARRIER_ISH)

#    define _ARM_BARRIER_ISH _ARM64_BARRIER_ISH

#   endif



static __inline int CRYPTO_UP_REF(volatile int *val, int *ret, void *lock)

{

    *ret = _InterlockedExchangeAdd_nf(val, 1) + 1;

    return 1;

}



static __inline int CRYPTO_DOWN_REF(volatile int *val, int *ret, void *lock)

{

    *ret = _InterlockedExchangeAdd_nf(val, -1) - 1;

    if (*ret == 0)

        __dmb(_ARM_BARRIER_ISH);

    return 1;

}

#  else

#   pragma intrinsic(_InterlockedExchangeAdd)



static __inline int CRYPTO_UP_REF(volatile int *val, int *ret, void *lock)

{

    *ret = _InterlockedExchangeAdd(val, 1) + 1;

    return 1;

}



static __inline int CRYPTO_DOWN_REF(volatile int *val, int *ret, void *lock)

{

    *ret = _InterlockedExchangeAdd(val, -1) - 1;

    return 1;

}

#  endif



# else



typedef int CRYPTO_REF_COUNT;