656 lines
24 KiB
C
656 lines
24 KiB
C
|
// Copyright 2021 The Abseil Authors.
|
||
|
//
|
||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
// you may not use this file except in compliance with the License.
|
||
|
// You may obtain a copy of the License at
|
||
|
//
|
||
|
// https://www.apache.org/licenses/LICENSE-2.0
|
||
|
//
|
||
|
// Unless required by applicable law or agreed to in writing, software
|
||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
// See the License for the specific language governing permissions and
|
||
|
// limitations under the License.
|
||
|
|
||
|
#ifndef ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_
|
||
|
#define ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_
|
||
|
|
||
|
#include <atomic>
|
||
|
#include <cassert>
|
||
|
#include <cstddef>
|
||
|
#include <cstdint>
|
||
|
#include <type_traits>
|
||
|
|
||
|
#include "absl/base/attributes.h"
|
||
|
#include "absl/base/config.h"
|
||
|
#include "absl/base/internal/endian.h"
|
||
|
#include "absl/base/internal/invoke.h"
|
||
|
#include "absl/base/optimization.h"
|
||
|
#include "absl/container/internal/compressed_tuple.h"
|
||
|
#include "absl/meta/type_traits.h"
|
||
|
#include "absl/strings/string_view.h"
|
||
|
|
||
|
namespace absl {
|
||
|
ABSL_NAMESPACE_BEGIN
|
||
|
namespace cord_internal {
|
||
|
|
||
|
// The overhead of a vtable is too much for Cord, so we roll our own subclasses
|
||
|
// using only a single byte to differentiate classes from each other - the "tag"
|
||
|
// byte. Define the subclasses first so we can provide downcasting helper
|
||
|
// functions in the base class.
|
||
|
struct CordRep;
|
||
|
struct CordRepConcat;
|
||
|
struct CordRepExternal;
|
||
|
struct CordRepFlat;
|
||
|
struct CordRepSubstring;
|
||
|
struct CordRepCrc;
|
||
|
class CordRepRing;
|
||
|
class CordRepBtree;
|
||
|
|
||
|
class CordzInfo;
|
||
|
|
||
|
// Default feature enable states for cord ring buffers
|
||
|
enum CordFeatureDefaults {
|
||
|
kCordEnableRingBufferDefault = false,
|
||
|
kCordShallowSubcordsDefault = false
|
||
|
};
|
||
|
|
||
|
extern std::atomic<bool> cord_ring_buffer_enabled;
|
||
|
extern std::atomic<bool> shallow_subcords_enabled;
|
||
|
|
||
|
// `cord_btree_exhaustive_validation` can be set to force exhaustive validation
|
||
|
// in debug assertions, and code that calls `IsValid()` explicitly. By default,
|
||
|
// assertions should be relatively cheap and AssertValid() can easily lead to
|
||
|
// O(n^2) complexity as recursive / full tree validation is O(n).
|
||
|
extern std::atomic<bool> cord_btree_exhaustive_validation;
|
||
|
|
||
|
inline void enable_cord_ring_buffer(bool enable) {
|
||
|
cord_ring_buffer_enabled.store(enable, std::memory_order_relaxed);
|
||
|
}
|
||
|
|
||
|
inline void enable_shallow_subcords(bool enable) {
|
||
|
shallow_subcords_enabled.store(enable, std::memory_order_relaxed);
|
||
|
}
|
||
|
|
||
|
enum Constants {
|
||
|
// The inlined size to use with absl::InlinedVector.
|
||
|
//
|
||
|
// Note: The InlinedVectors in this file (and in cord.h) do not need to use
|
||
|
// the same value for their inlined size. The fact that they do is historical.
|
||
|
// It may be desirable for each to use a different inlined size optimized for
|
||
|
// that InlinedVector's usage.
|
||
|
//
|
||
|
// TODO(jgm): Benchmark to see if there's a more optimal value than 47 for
|
||
|
// the inlined vector size (47 exists for backward compatibility).
|
||
|
kInlinedVectorSize = 47,
|
||
|
|
||
|
// Prefer copying blocks of at most this size, otherwise reference count.
|
||
|
kMaxBytesToCopy = 511
|
||
|
};
|
||
|
|
||
|
// Emits a fatal error "Unexpected node type: xyz" and aborts the program.
|
||
|
ABSL_ATTRIBUTE_NORETURN void LogFatalNodeType(CordRep* rep);
|
||
|
|
||
|
// Compact class for tracking the reference count and state flags for CordRep
|
||
|
// instances. Data is stored in an atomic int32_t for compactness and speed.
|
||
|
class RefcountAndFlags {
|
||
|
public:
|
||
|
constexpr RefcountAndFlags() : count_{kRefIncrement} {}
|
||
|
struct Immortal {};
|
||
|
explicit constexpr RefcountAndFlags(Immortal) : count_(kImmortalFlag) {}
|
||
|
|
||
|
// Increments the reference count. Imposes no memory ordering.
|
||
|
inline void Increment() {
|
||
|
count_.fetch_add(kRefIncrement, std::memory_order_relaxed);
|
||
|
}
|
||
|
|
||
|
// Asserts that the current refcount is greater than 0. If the refcount is
|
||
|
// greater than 1, decrements the reference count.
|
||
|
//
|
||
|
// Returns false if there are no references outstanding; true otherwise.
|
||
|
// Inserts barriers to ensure that state written before this method returns
|
||
|
// false will be visible to a thread that just observed this method returning
|
||
|
// false. Always returns false when the immortal bit is set.
|
||
|
inline bool Decrement() {
|
||
|
int32_t refcount = count_.load(std::memory_order_acquire) & kRefcountMask;
|
||
|
assert(refcount > 0 || refcount & kImmortalFlag);
|
||
|
return refcount != kRefIncrement &&
|
||
|
(count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) &
|
||
|
kRefcountMask) != kRefIncrement;
|
||
|
}
|
||
|
|
||
|
// Same as Decrement but expect that refcount is greater than 1.
|
||
|
inline bool DecrementExpectHighRefcount() {
|
||
|
int32_t refcount =
|
||
|
count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) &
|
||
|
kRefcountMask;
|
||
|
assert(refcount > 0 || refcount & kImmortalFlag);
|
||
|
return refcount != kRefIncrement;
|
||
|
}
|
||
|
|
||
|
// Returns the current reference count using acquire semantics.
|
||
|
inline int32_t Get() const {
|
||
|
return count_.load(std::memory_order_acquire) >> kNumFlags;
|
||
|
}
|
||
|
|
||
|
// Returns whether the atomic integer is 1.
|
||
|
// If the reference count is used in the conventional way, a
|
||
|
// reference count of 1 implies that the current thread owns the
|
||
|
// reference and no other thread shares it.
|
||
|
// This call performs the test for a reference count of one, and
|
||
|
// performs the memory barrier needed for the owning thread
|
||
|
// to act on the object, knowing that it has exclusive access to the
|
||
|
// object. Always returns false when the immortal bit is set.
|
||
|
inline bool IsOne() {
|
||
|
return (count_.load(std::memory_order_acquire) & kRefcountMask) ==
|
||
|
kRefIncrement;
|
||
|
}
|
||
|
|
||
|
bool IsImmortal() const {
|
||
|
return (count_.load(std::memory_order_relaxed) & kImmortalFlag) != 0;
|
||
|
}
|
||
|
|
||
|
private:
|
||
|
// We reserve the bottom bits for flags.
|
||
|
// kImmortalBit indicates that this entity should never be collected; it is
|
||
|
// used for the StringConstant constructor to avoid collecting immutable
|
||
|
// constant cords.
|
||
|
// kReservedFlag is reserved for future use.
|
||
|
enum Flags {
|
||
|
kNumFlags = 2,
|
||
|
|
||
|
kImmortalFlag = 0x1,
|
||
|
kReservedFlag = 0x2,
|
||
|
kRefIncrement = (1 << kNumFlags),
|
||
|
|
||
|
// Bitmask to use when checking refcount by equality. This masks out
|
||
|
// all flags except kImmortalFlag, which is part of the refcount for
|
||
|
// purposes of equality. (A refcount of 0 or 1 does not count as 0 or 1
|
||
|
// if the immortal bit is set.)
|
||
|
kRefcountMask = ~kReservedFlag,
|
||
|
};
|
||
|
|
||
|
std::atomic<int32_t> count_;
|
||
|
};
|
||
|
|
||
|
// Various representations that we allow
|
||
|
enum CordRepKind {
|
||
|
UNUSED_0 = 0,
|
||
|
SUBSTRING = 1,
|
||
|
CRC = 2,
|
||
|
BTREE = 3,
|
||
|
RING = 4,
|
||
|
EXTERNAL = 5,
|
||
|
|
||
|
// We have different tags for different sized flat arrays,
|
||
|
// starting with FLAT, and limited to MAX_FLAT_TAG. The below values map to an
|
||
|
// allocated range of 32 bytes to 256 KB. The current granularity is:
|
||
|
// - 8 byte granularity for flat sizes in [32 - 512]
|
||
|
// - 64 byte granularity for flat sizes in (512 - 8KiB]
|
||
|
// - 4KiB byte granularity for flat sizes in (8KiB, 256 KiB]
|
||
|
// If a new tag is needed in the future, then 'FLAT' and 'MAX_FLAT_TAG' should
|
||
|
// be adjusted as well as the Tag <---> Size mapping logic so that FLAT still
|
||
|
// represents the minimum flat allocation size. (32 bytes as of now).
|
||
|
FLAT = 6,
|
||
|
MAX_FLAT_TAG = 248
|
||
|
};
|
||
|
|
||
|
// There are various locations where we want to check if some rep is a 'plain'
|
||
|
// data edge, i.e. an external or flat rep. By having FLAT == EXTERNAL + 1, we
|
||
|
// can perform this check in a single branch as 'tag >= EXTERNAL'
|
||
|
// Likewise, we have some locations where we check for 'ring or external/flat',
|
||
|
// so likewise align RING to EXTERNAL.
|
||
|
// Note that we can leave this optimization to the compiler. The compiler will
|
||
|
// DTRT when it sees a condition like `tag == EXTERNAL || tag >= FLAT`.
|
||
|
static_assert(RING == BTREE + 1, "BTREE and RING not consecutive");
|
||
|
static_assert(EXTERNAL == RING + 1, "BTREE and EXTERNAL not consecutive");
|
||
|
static_assert(FLAT == EXTERNAL + 1, "EXTERNAL and FLAT not consecutive");
|
||
|
|
||
|
struct CordRep {
|
||
|
// Result from an `extract edge` operation. Contains the (possibly changed)
|
||
|
// tree node as well as the extracted edge, or {tree, nullptr} if no edge
|
||
|
// could be extracted.
|
||
|
// On success, the returned `tree` value is null if `extracted` was the only
|
||
|
// data edge inside the tree, a data edge if there were only two data edges in
|
||
|
// the tree, or the (possibly new / smaller) remaining tree with the extracted
|
||
|
// data edge removed.
|
||
|
struct ExtractResult {
|
||
|
CordRep* tree;
|
||
|
CordRep* extracted;
|
||
|
};
|
||
|
|
||
|
CordRep() = default;
|
||
|
constexpr CordRep(RefcountAndFlags::Immortal immortal, size_t l)
|
||
|
: length(l), refcount(immortal), tag(EXTERNAL), storage{} {}
|
||
|
|
||
|
// The following three fields have to be less than 32 bytes since
|
||
|
// that is the smallest supported flat node size.
|
||
|
size_t length;
|
||
|
RefcountAndFlags refcount;
|
||
|
// If tag < FLAT, it represents CordRepKind and indicates the type of node.
|
||
|
// Otherwise, the node type is CordRepFlat and the tag is the encoded size.
|
||
|
uint8_t tag;
|
||
|
|
||
|
// `storage` provides two main purposes:
|
||
|
// - the starting point for FlatCordRep.Data() [flexible-array-member]
|
||
|
// - 3 bytes of additional storage for use by derived classes.
|
||
|
// The latter is used by CordrepConcat and CordRepBtree. CordRepConcat stores
|
||
|
// a 'depth' value in storage[0], and the (future) CordRepBtree class stores
|
||
|
// `height`, `begin` and `end` in the 3 entries. Otherwise we would need to
|
||
|
// allocate room for these in the derived class, as not all compilers reuse
|
||
|
// padding space from the base class (clang and gcc do, MSVC does not, etc)
|
||
|
uint8_t storage[3];
|
||
|
|
||
|
// Returns true if this instance's tag matches the requested type.
|
||
|
constexpr bool IsRing() const { return tag == RING; }
|
||
|
constexpr bool IsSubstring() const { return tag == SUBSTRING; }
|
||
|
constexpr bool IsCrc() const { return tag == CRC; }
|
||
|
constexpr bool IsExternal() const { return tag == EXTERNAL; }
|
||
|
constexpr bool IsFlat() const { return tag >= FLAT; }
|
||
|
constexpr bool IsBtree() const { return tag == BTREE; }
|
||
|
|
||
|
inline CordRepRing* ring();
|
||
|
inline const CordRepRing* ring() const;
|
||
|
inline CordRepSubstring* substring();
|
||
|
inline const CordRepSubstring* substring() const;
|
||
|
inline CordRepCrc* crc();
|
||
|
inline const CordRepCrc* crc() const;
|
||
|
inline CordRepExternal* external();
|
||
|
inline const CordRepExternal* external() const;
|
||
|
inline CordRepFlat* flat();
|
||
|
inline const CordRepFlat* flat() const;
|
||
|
inline CordRepBtree* btree();
|
||
|
inline const CordRepBtree* btree() const;
|
||
|
|
||
|
// --------------------------------------------------------------------
|
||
|
// Memory management
|
||
|
|
||
|
// Destroys the provided `rep`.
|
||
|
static void Destroy(CordRep* rep);
|
||
|
|
||
|
// Increments the reference count of `rep`.
|
||
|
// Requires `rep` to be a non-null pointer value.
|
||
|
static inline CordRep* Ref(CordRep* rep);
|
||
|
|
||
|
// Decrements the reference count of `rep`. Destroys rep if count reaches
|
||
|
// zero. Requires `rep` to be a non-null pointer value.
|
||
|
static inline void Unref(CordRep* rep);
|
||
|
};
|
||
|
|
||
|
struct CordRepSubstring : public CordRep {
|
||
|
size_t start; // Starting offset of substring in child
|
||
|
CordRep* child;
|
||
|
|
||
|
// Creates a substring on `child`, adopting a reference on `child`.
|
||
|
// Requires `child` to be either a flat or external node, and `pos` and `n` to
|
||
|
// form a non-empty partial sub range of `'child`, i.e.:
|
||
|
// `n > 0 && n < length && n + pos <= length`
|
||
|
static inline CordRepSubstring* Create(CordRep* child, size_t pos, size_t n);
|
||
|
|
||
|
// Creates a substring of `rep`. Does not adopt a reference on `rep`.
|
||
|
// Requires `IsDataEdge(rep) && n > 0 && pos + n <= rep->length`.
|
||
|
// If `n == rep->length` then this method returns `CordRep::Ref(rep)`
|
||
|
// If `rep` is a substring of a flat or external node, then this method will
|
||
|
// return a new substring of that flat or external node with `pos` adjusted
|
||
|
// with the original `start` position.
|
||
|
static inline CordRep* Substring(CordRep* rep, size_t pos, size_t n);
|
||
|
};
|
||
|
|
||
|
// Type for function pointer that will invoke the releaser function and also
|
||
|
// delete the `CordRepExternalImpl` corresponding to the passed in
|
||
|
// `CordRepExternal`.
|
||
|
using ExternalReleaserInvoker = void (*)(CordRepExternal*);
|
||
|
|
||
|
// External CordReps are allocated together with a type erased releaser. The
|
||
|
// releaser is stored in the memory directly following the CordRepExternal.
|
||
|
struct CordRepExternal : public CordRep {
|
||
|
CordRepExternal() = default;
|
||
|
explicit constexpr CordRepExternal(absl::string_view str)
|
||
|
: CordRep(RefcountAndFlags::Immortal{}, str.size()),
|
||
|
base(str.data()),
|
||
|
releaser_invoker(nullptr) {}
|
||
|
|
||
|
const char* base;
|
||
|
// Pointer to function that knows how to call and destroy the releaser.
|
||
|
ExternalReleaserInvoker releaser_invoker;
|
||
|
|
||
|
// Deletes (releases) the external rep.
|
||
|
// Requires rep != nullptr and rep->IsExternal()
|
||
|
static void Delete(CordRep* rep);
|
||
|
};
|
||
|
|
||
|
struct Rank1 {};
|
||
|
struct Rank0 : Rank1 {};
|
||
|
|
||
|
template <typename Releaser, typename = ::absl::base_internal::invoke_result_t<
|
||
|
Releaser, absl::string_view>>
|
||
|
void InvokeReleaser(Rank0, Releaser&& releaser, absl::string_view data) {
|
||
|
::absl::base_internal::invoke(std::forward<Releaser>(releaser), data);
|
||
|
}
|
||
|
|
||
|
template <typename Releaser,
|
||
|
typename = ::absl::base_internal::invoke_result_t<Releaser>>
|
||
|
void InvokeReleaser(Rank1, Releaser&& releaser, absl::string_view) {
|
||
|
::absl::base_internal::invoke(std::forward<Releaser>(releaser));
|
||
|
}
|
||
|
|
||
|
// We use CompressedTuple so that we can benefit from EBCO.
|
||
|
template <typename Releaser>
|
||
|
struct CordRepExternalImpl
|
||
|
: public CordRepExternal,
|
||
|
public ::absl::container_internal::CompressedTuple<Releaser> {
|
||
|
// The extra int arg is so that we can avoid interfering with copy/move
|
||
|
// constructors while still benefitting from perfect forwarding.
|
||
|
template <typename T>
|
||
|
CordRepExternalImpl(T&& releaser, int)
|
||
|
: CordRepExternalImpl::CompressedTuple(std::forward<T>(releaser)) {
|
||
|
this->releaser_invoker = &Release;
|
||
|
}
|
||
|
|
||
|
~CordRepExternalImpl() {
|
||
|
InvokeReleaser(Rank0{}, std::move(this->template get<0>()),
|
||
|
absl::string_view(base, length));
|
||
|
}
|
||
|
|
||
|
static void Release(CordRepExternal* rep) {
|
||
|
delete static_cast<CordRepExternalImpl*>(rep);
|
||
|
}
|
||
|
};
|
||
|
|
||
|
inline CordRepSubstring* CordRepSubstring::Create(CordRep* child, size_t pos,
|
||
|
size_t n) {
|
||
|
assert(child != nullptr);
|
||
|
assert(n > 0);
|
||
|
assert(n < child->length);
|
||
|
assert(pos < child->length);
|
||
|
assert(n <= child->length - pos);
|
||
|
|
||
|
// TODO(b/217376272): Harden internal logic.
|
||
|
// Move to strategical places inside the Cord logic and make this an assert.
|
||
|
if (ABSL_PREDICT_FALSE(!(child->IsExternal() || child->IsFlat()))) {
|
||
|
LogFatalNodeType(child);
|
||
|
}
|
||
|
|
||
|
CordRepSubstring* rep = new CordRepSubstring();
|
||
|
rep->length = n;
|
||
|
rep->tag = SUBSTRING;
|
||
|
rep->start = pos;
|
||
|
rep->child = child;
|
||
|
return rep;
|
||
|
}
|
||
|
|
||
|
inline CordRep* CordRepSubstring::Substring(CordRep* rep, size_t pos,
|
||
|
size_t n) {
|
||
|
assert(rep != nullptr);
|
||
|
assert(n != 0);
|
||
|
assert(pos < rep->length);
|
||
|
assert(n <= rep->length - pos);
|
||
|
if (n == rep->length) return CordRep::Ref(rep);
|
||
|
if (rep->IsSubstring()) {
|
||
|
pos += rep->substring()->start;
|
||
|
rep = rep->substring()->child;
|
||
|
}
|
||
|
CordRepSubstring* substr = new CordRepSubstring();
|
||
|
substr->length = n;
|
||
|
substr->tag = SUBSTRING;
|
||
|
substr->start = pos;
|
||
|
substr->child = CordRep::Ref(rep);
|
||
|
return substr;
|
||
|
}
|
||
|
|
||
|
inline void CordRepExternal::Delete(CordRep* rep) {
|
||
|
assert(rep != nullptr && rep->IsExternal());
|
||
|
auto* rep_external = static_cast<CordRepExternal*>(rep);
|
||
|
assert(rep_external->releaser_invoker != nullptr);
|
||
|
rep_external->releaser_invoker(rep_external);
|
||
|
}
|
||
|
|
||
|
template <typename Str>
|
||
|
struct ConstInitExternalStorage {
|
||
|
ABSL_CONST_INIT static CordRepExternal value;
|
||
|
};
|
||
|
|
||
|
template <typename Str>
|
||
|
ABSL_CONST_INIT CordRepExternal
|
||
|
ConstInitExternalStorage<Str>::value(Str::value);
|
||
|
|
||
|
enum {
|
||
|
kMaxInline = 15,
|
||
|
};
|
||
|
|
||
|
constexpr char GetOrNull(absl::string_view data, size_t pos) {
|
||
|
return pos < data.size() ? data[pos] : '\0';
|
||
|
}
|
||
|
|
||
|
// We store cordz_info as 64 bit pointer value in big endian format. This
|
||
|
// guarantees that the least significant byte of cordz_info matches the last
|
||
|
// byte of the inline data representation in as_chars_, which holds the inlined
|
||
|
// size or the 'is_tree' bit.
|
||
|
using cordz_info_t = int64_t;
|
||
|
|
||
|
// Assert that the `cordz_info` pointer value perfectly overlaps the last half
|
||
|
// of `as_chars_` and can hold a pointer value.
|
||
|
static_assert(sizeof(cordz_info_t) * 2 == kMaxInline + 1, "");
|
||
|
static_assert(sizeof(cordz_info_t) >= sizeof(intptr_t), "");
|
||
|
|
||
|
// BigEndianByte() creates a big endian representation of 'value', i.e.: a big
|
||
|
// endian value where the last byte in the host's representation holds 'value`,
|
||
|
// with all other bytes being 0.
|
||
|
static constexpr cordz_info_t BigEndianByte(unsigned char value) {
|
||
|
#if defined(ABSL_IS_BIG_ENDIAN)
|
||
|
return value;
|
||
|
#else
|
||
|
return static_cast<cordz_info_t>(value) << ((sizeof(cordz_info_t) - 1) * 8);
|
||
|
#endif
|
||
|
}
|
||
|
|
||
|
class InlineData {
|
||
|
public:
|
||
|
// DefaultInitType forces the use of the default initialization constructor.
|
||
|
enum DefaultInitType { kDefaultInit };
|
||
|
|
||
|
// kNullCordzInfo holds the big endian representation of intptr_t(1)
|
||
|
// This is the 'null' / initial value of 'cordz_info'. The null value
|
||
|
// is specifically big endian 1 as with 64-bit pointers, the last
|
||
|
// byte of cordz_info overlaps with the last byte holding the tag.
|
||
|
static constexpr cordz_info_t kNullCordzInfo = BigEndianByte(1);
|
||
|
|
||
|
constexpr InlineData() : as_chars_{0} {}
|
||
|
explicit InlineData(DefaultInitType) {}
|
||
|
explicit constexpr InlineData(CordRep* rep) : as_tree_(rep) {}
|
||
|
explicit constexpr InlineData(absl::string_view chars)
|
||
|
: as_chars_{
|
||
|
GetOrNull(chars, 0), GetOrNull(chars, 1),
|
||
|
GetOrNull(chars, 2), GetOrNull(chars, 3),
|
||
|
GetOrNull(chars, 4), GetOrNull(chars, 5),
|
||
|
GetOrNull(chars, 6), GetOrNull(chars, 7),
|
||
|
GetOrNull(chars, 8), GetOrNull(chars, 9),
|
||
|
GetOrNull(chars, 10), GetOrNull(chars, 11),
|
||
|
GetOrNull(chars, 12), GetOrNull(chars, 13),
|
||
|
GetOrNull(chars, 14), static_cast<char>((chars.size() << 1))} {}
|
||
|
|
||
|
// Returns true if the current instance is empty.
|
||
|
// The 'empty value' is an inlined data value of zero length.
|
||
|
bool is_empty() const { return tag() == 0; }
|
||
|
|
||
|
// Returns true if the current instance holds a tree value.
|
||
|
bool is_tree() const { return (tag() & 1) != 0; }
|
||
|
|
||
|
// Returns true if the current instance holds a cordz_info value.
|
||
|
// Requires the current instance to hold a tree value.
|
||
|
bool is_profiled() const {
|
||
|
assert(is_tree());
|
||
|
return as_tree_.cordz_info != kNullCordzInfo;
|
||
|
}
|
||
|
|
||
|
// Returns true if either of the provided instances hold a cordz_info value.
|
||
|
// This method is more efficient than the equivalent `data1.is_profiled() ||
|
||
|
// data2.is_profiled()`. Requires both arguments to hold a tree.
|
||
|
static bool is_either_profiled(const InlineData& data1,
|
||
|
const InlineData& data2) {
|
||
|
assert(data1.is_tree() && data2.is_tree());
|
||
|
return (data1.as_tree_.cordz_info | data2.as_tree_.cordz_info) !=
|
||
|
kNullCordzInfo;
|
||
|
}
|
||
|
|
||
|
// Returns the cordz_info sampling instance for this instance, or nullptr
|
||
|
// if the current instance is not sampled and does not have CordzInfo data.
|
||
|
// Requires the current instance to hold a tree value.
|
||
|
CordzInfo* cordz_info() const {
|
||
|
assert(is_tree());
|
||
|
intptr_t info = static_cast<intptr_t>(
|
||
|
absl::big_endian::ToHost64(static_cast<uint64_t>(as_tree_.cordz_info)));
|
||
|
assert(info & 1);
|
||
|
return reinterpret_cast<CordzInfo*>(info - 1);
|
||
|
}
|
||
|
|
||
|
// Sets the current cordz_info sampling instance for this instance, or nullptr
|
||
|
// if the current instance is not sampled and does not have CordzInfo data.
|
||
|
// Requires the current instance to hold a tree value.
|
||
|
void set_cordz_info(CordzInfo* cordz_info) {
|
||
|
assert(is_tree());
|
||
|
uintptr_t info = reinterpret_cast<uintptr_t>(cordz_info) | 1;
|
||
|
as_tree_.cordz_info =
|
||
|
static_cast<cordz_info_t>(absl::big_endian::FromHost64(info));
|
||
|
}
|
||
|
|
||
|
// Resets the current cordz_info to null / empty.
|
||
|
void clear_cordz_info() {
|
||
|
assert(is_tree());
|
||
|
as_tree_.cordz_info = kNullCordzInfo;
|
||
|
}
|
||
|
|
||
|
// Returns a read only pointer to the character data inside this instance.
|
||
|
// Requires the current instance to hold inline data.
|
||
|
const char* as_chars() const {
|
||
|
assert(!is_tree());
|
||
|
return as_chars_;
|
||
|
}
|
||
|
|
||
|
// Returns a mutable pointer to the character data inside this instance.
|
||
|
// Should be used for 'write only' operations setting an inlined value.
|
||
|
// Applications can set the value of inlined data either before or after
|
||
|
// setting the inlined size, i.e., both of the below are valid:
|
||
|
//
|
||
|
// // Set inlined data and inline size
|
||
|
// memcpy(data_.as_chars(), data, size);
|
||
|
// data_.set_inline_size(size);
|
||
|
//
|
||
|
// // Set inlined size and inline data
|
||
|
// data_.set_inline_size(size);
|
||
|
// memcpy(data_.as_chars(), data, size);
|
||
|
//
|
||
|
// It's an error to read from the returned pointer without a preceding write
|
||
|
// if the current instance does not hold inline data, i.e.: is_tree() == true.
|
||
|
char* as_chars() { return as_chars_; }
|
||
|
|
||
|
// Returns the tree value of this value.
|
||
|
// Requires the current instance to hold a tree value.
|
||
|
CordRep* as_tree() const {
|
||
|
assert(is_tree());
|
||
|
return as_tree_.rep;
|
||
|
}
|
||
|
|
||
|
// Initialize this instance to holding the tree value `rep`,
|
||
|
// initializing the cordz_info to null, i.e.: 'not profiled'.
|
||
|
void make_tree(CordRep* rep) {
|
||
|
as_tree_.rep = rep;
|
||
|
as_tree_.cordz_info = kNullCordzInfo;
|
||
|
}
|
||
|
|
||
|
// Set the tree value of this instance to 'rep`.
|
||
|
// Requires the current instance to already hold a tree value.
|
||
|
// Does not affect the value of cordz_info.
|
||
|
void set_tree(CordRep* rep) {
|
||
|
assert(is_tree());
|
||
|
as_tree_.rep = rep;
|
||
|
}
|
||
|
|
||
|
// Returns the size of the inlined character data inside this instance.
|
||
|
// Requires the current instance to hold inline data.
|
||
|
size_t inline_size() const {
|
||
|
assert(!is_tree());
|
||
|
return tag() >> 1;
|
||
|
}
|
||
|
|
||
|
// Sets the size of the inlined character data inside this instance.
|
||
|
// Requires `size` to be <= kMaxInline.
|
||
|
// See the documentation on 'as_chars()' for more information and examples.
|
||
|
void set_inline_size(size_t size) {
|
||
|
ABSL_ASSERT(size <= kMaxInline);
|
||
|
tag() = static_cast<char>(size << 1);
|
||
|
}
|
||
|
|
||
|
private:
|
||
|
// See cordz_info_t for forced alignment and size of `cordz_info` details.
|
||
|
struct AsTree {
|
||
|
explicit constexpr AsTree(absl::cord_internal::CordRep* tree)
|
||
|
: rep(tree), cordz_info(kNullCordzInfo) {}
|
||
|
// This union uses up extra space so that whether rep is 32 or 64 bits,
|
||
|
// cordz_info will still start at the eighth byte, and the last
|
||
|
// byte of cordz_info will still be the last byte of InlineData.
|
||
|
union {
|
||
|
absl::cord_internal::CordRep* rep;
|
||
|
cordz_info_t unused_aligner;
|
||
|
};
|
||
|
cordz_info_t cordz_info;
|
||
|
};
|
||
|
|
||
|
char& tag() { return reinterpret_cast<char*>(this)[kMaxInline]; }
|
||
|
char tag() const { return reinterpret_cast<const char*>(this)[kMaxInline]; }
|
||
|
|
||
|
// If the data has length <= kMaxInline, we store it in `as_chars_`, and
|
||
|
// store the size in the last char of `as_chars_` shifted left + 1.
|
||
|
// Else we store it in a tree and store a pointer to that tree in
|
||
|
// `as_tree_.rep` and store a tag in `tagged_size`.
|
||
|
union {
|
||
|
char as_chars_[kMaxInline + 1];
|
||
|
AsTree as_tree_;
|
||
|
};
|
||
|
};
|
||
|
|
||
|
static_assert(sizeof(InlineData) == kMaxInline + 1, "");
|
||
|
|
||
|
inline CordRepSubstring* CordRep::substring() {
|
||
|
assert(IsSubstring());
|
||
|
return static_cast<CordRepSubstring*>(this);
|
||
|
}
|
||
|
|
||
|
inline const CordRepSubstring* CordRep::substring() const {
|
||
|
assert(IsSubstring());
|
||
|
return static_cast<const CordRepSubstring*>(this);
|
||
|
}
|
||
|
|
||
|
inline CordRepExternal* CordRep::external() {
|
||
|
assert(IsExternal());
|
||
|
return static_cast<CordRepExternal*>(this);
|
||
|
}
|
||
|
|
||
|
inline const CordRepExternal* CordRep::external() const {
|
||
|
assert(IsExternal());
|
||
|
return static_cast<const CordRepExternal*>(this);
|
||
|
}
|
||
|
|
||
|
inline CordRep* CordRep::Ref(CordRep* rep) {
|
||
|
// ABSL_ASSUME is a workaround for
|
||
|
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105585
|
||
|
ABSL_ASSUME(rep != nullptr);
|
||
|
rep->refcount.Increment();
|
||
|
return rep;
|
||
|
}
|
||
|
|
||
|
inline void CordRep::Unref(CordRep* rep) {
|
||
|
assert(rep != nullptr);
|
||
|
// Expect refcount to be 0. Avoiding the cost of an atomic decrement should
|
||
|
// typically outweigh the cost of an extra branch checking for ref == 1.
|
||
|
if (ABSL_PREDICT_FALSE(!rep->refcount.DecrementExpectHighRefcount())) {
|
||
|
Destroy(rep);
|
||
|
}
|
||
|
}
|
||
|
|
||
|
} // namespace cord_internal
|
||
|
|
||
|
ABSL_NAMESPACE_END
|
||
|
} // namespace absl
|
||
|
#endif // ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_
|