blob: 3f700482695fdd70e64cee792ccd868777c30938 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
|
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "third_party/base/allocator/partition_allocator/partition_page.h"
#include "third_party/base/allocator/partition_allocator/partition_direct_map_extent.h"
#include "third_party/base/allocator/partition_allocator/partition_root_base.h"
namespace pdfium {
namespace base {
namespace internal {
namespace {
ALWAYS_INLINE void PartitionDirectUnmap(PartitionPage* page) {
PartitionRootBase* root = PartitionRootBase::FromPage(page);
const PartitionDirectMapExtent* extent =
PartitionDirectMapExtent::FromPage(page);
size_t unmap_size = extent->map_size;
// Maintain the doubly-linked list of all direct mappings.
if (extent->prev_extent) {
DCHECK(extent->prev_extent->next_extent == extent);
extent->prev_extent->next_extent = extent->next_extent;
} else {
root->direct_map_list = extent->next_extent;
}
if (extent->next_extent) {
DCHECK(extent->next_extent->prev_extent == extent);
extent->next_extent->prev_extent = extent->prev_extent;
}
// Add on the size of the trailing guard page and preceeding partition
// page.
unmap_size += kPartitionPageSize + kSystemPageSize;
size_t uncommitted_page_size = page->bucket->slot_size + kSystemPageSize;
root->DecreaseCommittedPages(uncommitted_page_size);
DCHECK(root->total_size_of_direct_mapped_pages >= uncommitted_page_size);
root->total_size_of_direct_mapped_pages -= uncommitted_page_size;
DCHECK(!(unmap_size & kPageAllocationGranularityOffsetMask));
char* ptr = reinterpret_cast<char*>(PartitionPage::ToPointer(page));
// Account for the mapping starting a partition page before the actual
// allocation address.
ptr -= kPartitionPageSize;
FreePages(ptr, unmap_size);
}
ALWAYS_INLINE void PartitionRegisterEmptyPage(PartitionPage* page) {
DCHECK(page->is_empty());
PartitionRootBase* root = PartitionRootBase::FromPage(page);
// If the page is already registered as empty, give it another life.
if (page->empty_cache_index != -1) {
DCHECK(page->empty_cache_index >= 0);
DCHECK(static_cast<unsigned>(page->empty_cache_index) < kMaxFreeableSpans);
DCHECK(root->global_empty_page_ring[page->empty_cache_index] == page);
root->global_empty_page_ring[page->empty_cache_index] = nullptr;
}
int16_t current_index = root->global_empty_page_ring_index;
PartitionPage* page_to_decommit = root->global_empty_page_ring[current_index];
// The page might well have been re-activated, filled up, etc. before we get
// around to looking at it here.
if (page_to_decommit)
page_to_decommit->DecommitIfPossible(root);
// We put the empty slot span on our global list of "pages that were once
// empty". thus providing it a bit of breathing room to get re-used before
// we really free it. This improves performance, particularly on Mac OS X
// which has subpar memory management performance.
root->global_empty_page_ring[current_index] = page;
page->empty_cache_index = current_index;
++current_index;
if (current_index == kMaxFreeableSpans)
current_index = 0;
root->global_empty_page_ring_index = current_index;
}
} // namespace
// static
PartitionPage PartitionPage::sentinel_page_;
PartitionPage* PartitionPage::get_sentinel_page() {
return &sentinel_page_;
}
void PartitionPage::FreeSlowPath() {
DCHECK(this != get_sentinel_page());
if (LIKELY(this->num_allocated_slots == 0)) {
// Page became fully unused.
if (UNLIKELY(bucket->is_direct_mapped())) {
PartitionDirectUnmap(this);
return;
}
// If it's the current active page, change it. We bounce the page to
// the empty list as a force towards defragmentation.
if (LIKELY(this == bucket->active_pages_head))
bucket->SetNewActivePage();
DCHECK(bucket->active_pages_head != this);
set_raw_size(0);
DCHECK(!get_raw_size());
PartitionRegisterEmptyPage(this);
} else {
DCHECK(!bucket->is_direct_mapped());
// Ensure that the page is full. That's the only valid case if we
// arrive here.
DCHECK(this->num_allocated_slots < 0);
// A transition of num_allocated_slots from 0 to -1 is not legal, and
// likely indicates a double-free.
CHECK(this->num_allocated_slots != -1);
this->num_allocated_slots = -this->num_allocated_slots - 2;
DCHECK(this->num_allocated_slots == bucket->get_slots_per_span() - 1);
// Fully used page became partially used. It must be put back on the
// non-full page list. Also make it the current page to increase the
// chances of it being filled up again. The old current page will be
// the next page.
DCHECK(!this->next_page);
if (LIKELY(bucket->active_pages_head != get_sentinel_page()))
this->next_page = bucket->active_pages_head;
bucket->active_pages_head = this;
--bucket->num_full_pages;
// Special case: for a partition page with just a single slot, it may
// now be empty and we want to run it through the empty logic.
if (UNLIKELY(this->num_allocated_slots == 0))
FreeSlowPath();
}
}
void PartitionPage::Decommit(PartitionRootBase* root) {
DCHECK(is_empty());
DCHECK(!bucket->is_direct_mapped());
void* addr = PartitionPage::ToPointer(this);
root->DecommitSystemPages(addr, bucket->get_bytes_per_span());
// We actually leave the decommitted page in the active list. We'll sweep
// it on to the decommitted page list when we next walk the active page
// list.
// Pulling this trick enables us to use a singly-linked page list for all
// cases, which is critical in keeping the page metadata structure down to
// 32 bytes in size.
freelist_head = nullptr;
num_unprovisioned_slots = 0;
DCHECK(is_decommitted());
}
void PartitionPage::DecommitIfPossible(PartitionRootBase* root) {
DCHECK(empty_cache_index >= 0);
DCHECK(static_cast<unsigned>(empty_cache_index) < kMaxFreeableSpans);
DCHECK(this == root->global_empty_page_ring[empty_cache_index]);
empty_cache_index = -1;
if (is_empty())
Decommit(root);
}
} // namespace internal
} // namespace base
} // namespace pdfium
|
