summaryrefslogtreecommitdiff
path: root/src/mem/qos/mem_ctrl.hh
blob: e31f21b234ee5c162cfd61254facfa808c1ee7fc (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
/*
 * Copyright (c) 2019 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Matteo Andreozzi
 */

#include "debug/QOS.hh"
#include "mem/abstract_mem.hh"
#include "mem/qos/q_policy.hh"
#include "mem/qos/policy.hh"
#include "params/QoSMemCtrl.hh"
#include "sim/system.hh"

#include <unordered_map>
#include <vector>
#include <deque>

#ifndef __MEM_QOS_MEM_CTRL_HH__
#define __MEM_QOS_MEM_CTRL_HH__

namespace QoS {

/**
 * The QoS::MemCtrl is a base class for Memory objects
 * which support QoS - it provides access to a set of QoS
 * scheduling policies
 */
class MemCtrl: public AbstractMemory
{
  public:
    /** Bus Direction */
    enum BusState { READ, WRITE };

  protected:
    /** QoS Policy, assigns QoS priority to the incoming packets */
    const std::unique_ptr<Policy> policy;

    /** QoS Bus Turnaround Policy: selects the bus direction (READ/WRITE) */
    const std::unique_ptr<TurnaroundPolicy> turnPolicy;

    /** QoS Queue Policy: selects packet among same-priority queue */
    const std::unique_ptr<QueuePolicy> queuePolicy;

    /** Number of configured QoS priorities */
    const uint8_t _numPriorities;

    /** Enables QoS priority escalation */
    const bool qosPriorityEscalation;

    /**
     * Enables QoS synchronized scheduling invokes the QoS scheduler
     * on all masters, at every packet arrival.
     */
    const bool qosSyncroScheduler;

    /** Hash of master ID - master name */
    std::unordered_map<MasterID, const std::string> masters;

    /** Hash of masters - number of packets queued per priority */
    std::unordered_map<MasterID, std::vector<uint64_t> > packetPriorities;

    /** Hash of masters - address of request - queue of times of request */
    std::unordered_map<MasterID,
            std::unordered_map<uint64_t, std::deque<uint64_t>> > requestTimes;

    /**
     * Vector of QoS priorities/last service time. Refreshed at every
     * qosSchedule call.
     */
    std::vector<Tick> serviceTick;

    /** Read request packets queue length in #packets, per QoS priority */
    std::vector<uint64_t> readQueueSizes;

    /** Write request packets queue length in #packets, per QoS priority */
    std::vector<uint64_t> writeQueueSizes;

    /** Total read request packets queue length in #packets */
    uint64_t totalReadQueueSize;

    /** Total write request packets queue length in #packets */
    uint64_t totalWriteQueueSize;

    /**
     * Bus state used to control the read/write switching and drive
     * the scheduling of the next request.
     */
    BusState busState;

    /** bus state for next request event triggered */
    BusState busStateNext;

    struct MemCtrlStats : public Stats::Group
    {
        MemCtrlStats(MemCtrl &mc);

        void regStats() override;

        const MemCtrl &memCtrl;

        /** per-master average QoS priority */
        Stats::VectorStandardDeviation avgPriority;
        /**
         * per-master average QoS distance between assigned and
         * queued values
         */
        Stats::VectorStandardDeviation avgPriorityDistance;

        /** per-priority minimum latency */
        Stats::Vector priorityMinLatency;
        /** per-priority maximum latency */
        Stats::Vector priorityMaxLatency;
        /** Count the number of turnarounds READ to WRITE */
        Stats::Scalar numReadWriteTurnArounds;
        /** Count the number of turnarounds WRITE to READ */
        Stats::Scalar numWriteReadTurnArounds;
        /** Count the number of times bus staying in READ state */
        Stats::Scalar numStayReadState;
        /** Count the number of times bus staying in WRITE state */
        Stats::Scalar numStayWriteState;
    } stats;

    /**
     * Initializes dynamically counters and
     * statistics for a given Master
     *
     * @param m_id the master ID
     */
    void addMaster(const MasterID m_id);

    /**
     * Called upon receiving a request or
     * updates statistics and updates queues status
     *
     * @param dir request direction
     * @param m_id master id
     * @param qos packet qos value
     * @param addr packet address
     * @param entries number of entries to record
     */
    void logRequest(BusState dir, MasterID m_id, uint8_t qos,
                    Addr addr, uint64_t entries);

    /**
     * Called upon receiving a response,
     * updates statistics and updates queues status
     *
     * @param dir response direction
     * @param m_id master id
     * @param qos packet qos value
     * @param addr packet address
     * @param entries number of entries to record
     * @param delay response delay
     */
    void logResponse(BusState dir, MasterID m_id, uint8_t qos,
                     Addr addr, uint64_t entries, double delay);

    /**
     * Assign priority to a packet by executing
     * the configured QoS policy.
     *
     * @param queues_ptr list of pointers to packet queues
     * @param queue_entry_size size in bytes per each packet in the queue
     * @param pkt pointer to the Packet
     * @return a QoS priority value
     */
    template<typename Queues>
    uint8_t qosSchedule(std::initializer_list<Queues*> queues_ptr,
                        uint64_t queue_entry_size, const PacketPtr pkt);

    using SimObject::schedule;
    uint8_t schedule(MasterID m_id, uint64_t data);
    uint8_t schedule(const PacketPtr pkt);

    /**
     * Returns next bus direction (READ or WRITE)
     * based on configured policy.
     */
    BusState selectNextBusState();

    /**
     * Set current bus direction (READ or WRITE)
     * from next selected one
     */
    void setCurrentBusState() { busState = busStateNext; }

    /**
     * Record statistics on turnarounds based on
     * busStateNext and busState values
     */
    void recordTurnaroundStats();

    /**
     * Escalates/demotes priority of all packets
     * belonging to the passed master to given
     * priority value
     *
     * @param queues list of pointers to packet queues
     * @param queue_entry_size size of an entry in the queue
     * @param m_id master whose packets priority will change
     * @param tgt_prio target priority value
     */
    template<typename Queues>
    void escalate(std::initializer_list<Queues*> queues,
                  uint64_t queue_entry_size,
                  MasterID m_id, uint8_t tgt_prio);

    /**
     * Escalates/demotes priority of all packets
     * belonging to the passed master to given
     * priority value in a specified cluster of queues
     * (e.g. read queues or write queues) which is passed
     * as an argument to the function.
     * The curr_prio/tgt_prio parameters are queue selectors in the
     * queue cluster.
     *
     * @param queues reference to packet queues
     * @param queue_entry_size size of an entry in the queue
     * @param m_id master whose packets priority will change
     * @param curr_prio source queue priority value
     * @param tgt_prio target queue priority value
     */
    template<typename Queues>
    void escalateQueues(Queues& queues, uint64_t queue_entry_size,
                        MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio);

  public:
    /**
     * QoS Memory base class
     *
     * @param p pointer to QoSMemCtrl parameters
     */
    MemCtrl(const QoSMemCtrlParams*);

    virtual ~MemCtrl();

    /**
     * Initializes this object
     */
    void init() override;

    /**
     * Gets the current bus state
     *
     * @return current bus state
     */
    BusState getBusState() const { return busState; }

    /**
     * Gets the next bus state
     *
     * @return next bus state
     */
    BusState getBusStateNext() const { return busStateNext; }

    /**
     * hasMaster returns true if the selected master(ID) has
     * been registered in the memory controller, which happens if
     * the memory controller has received at least a packet from
     * that master.
     *
     * @param m_id master id to lookup
     * @return true if the memory controller has received a packet
     *         from the master, false otherwise.
     */
    bool hasMaster(MasterID m_id) const
    {
        return masters.find(m_id) != masters.end();
    }

    /**
     * Gets a READ queue size
     *
     * @param prio QoS Priority of the queue
     * @return queue size in packets
     */
    uint64_t getReadQueueSize(const uint8_t prio) const
    { return readQueueSizes[prio]; }

    /**
     * Gets a WRITE queue size
     *
     * @param prio QoS Priority of the queue
     * @return queue size in packets
     */
    uint64_t getWriteQueueSize(const uint8_t prio) const
    { return writeQueueSizes[prio]; }

    /**
     * Gets the total combined READ queues size
     *
     * @return total queues size in packets
     */
    uint64_t getTotalReadQueueSize() const { return totalReadQueueSize; }

    /**
     * Gets the total combined WRITE queues size
     *
     * @return total queues size in packets
     */
    uint64_t getTotalWriteQueueSize() const { return totalWriteQueueSize; }

    /**
     * Gets the last service tick related to a QoS Priority
     *
     * @param prio QoS Priority
     * @return tick
     */
    Tick getServiceTick(const uint8_t prio) const { return serviceTick[prio]; }

    /**
     * Gets the total number of priority levels in the
     * QoS memory controller.
     *
     * @return total number of priority levels
     */
    uint8_t numPriorities() const { return _numPriorities; }
};

template<typename Queues>
void
MemCtrl::escalateQueues(Queues& queues, uint64_t queue_entry_size,
                        MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio)
{
    auto it = queues[curr_prio].begin();
    while (it != queues[curr_prio].end()) {
        // No packets left to move
        if (packetPriorities[m_id][curr_prio] == 0)
            break;

        auto pkt = *it;

        DPRINTF(QOS,
                "QoSMemCtrl::escalate checking priority %d packet "
                "m_id %d address %d\n", curr_prio,
                pkt->masterId(), pkt->getAddr());

        // Found a packet to move
        if (pkt->masterId() == m_id) {

            uint64_t moved_entries = divCeil(pkt->getSize(),
                                             queue_entry_size);

            DPRINTF(QOS,
                    "QoSMemCtrl::escalate Master %s [id %d] moving "
                    "packet addr %d size %d (p size %d) from priority %d "
                    "to priority %d - "
                    "this master packets %d (entries to move %d)\n",
                    masters[m_id], m_id, pkt->getAddr(),
                    pkt->getSize(),
                    queue_entry_size, curr_prio, tgt_prio,
                    packetPriorities[m_id][curr_prio], moved_entries);


            if (pkt->isRead()) {
                panic_if(readQueueSizes[curr_prio] < moved_entries,
                         "QoSMemCtrl::escalate master %s negative READ "
                         "packets for priority %d",
                        masters[m_id], tgt_prio);
                readQueueSizes[curr_prio] -= moved_entries;
                readQueueSizes[tgt_prio] += moved_entries;
            } else if (pkt->isWrite()) {
                panic_if(writeQueueSizes[curr_prio] < moved_entries,
                         "QoSMemCtrl::escalate master %s negative WRITE "
                         "packets for priority %d",
                        masters[m_id], tgt_prio);
                writeQueueSizes[curr_prio] -= moved_entries;
                writeQueueSizes[tgt_prio] += moved_entries;
            }

            // Change QoS priority and move packet
            pkt->qosValue(tgt_prio);
            queues[tgt_prio].push_back(pkt);

            // Erase element from source packet queue, this will
            // increment the iterator
            it = queues[curr_prio].erase(it);
            panic_if(packetPriorities[m_id][curr_prio] < moved_entries,
                     "QoSMemCtrl::escalate master %s negative packets "
                     "for priority %d",
                     masters[m_id], tgt_prio);

            packetPriorities[m_id][curr_prio] -= moved_entries;
            packetPriorities[m_id][tgt_prio] += moved_entries;
        } else {
            // Increment iterator to next location in the queue
            it++;
        }
    }
}

template<typename Queues>
void
MemCtrl::escalate(std::initializer_list<Queues*> queues,
                  uint64_t queue_entry_size,
                  MasterID m_id, uint8_t tgt_prio)
{
    // If needed, initialize all counters and statistics
    // for this master
    addMaster(m_id);

    DPRINTF(QOS,
            "QoSMemCtrl::escalate Master %s [id %d] to priority "
            "%d (currently %d packets)\n",masters[m_id], m_id, tgt_prio,
            packetPriorities[m_id][tgt_prio]);

    for (uint8_t curr_prio = 0; curr_prio < numPriorities(); ++curr_prio) {
        // Skip target priority
        if (curr_prio == tgt_prio)
            continue;

        // Process other priority packet
        while (packetPriorities[m_id][curr_prio] > 0) {
            DPRINTF(QOS,
                    "QoSMemCtrl::escalate MID %d checking priority %d "
                    "(packets %d)- current packets in prio %d:  %d\n"
                    "\t(source read %d source write %d target read %d, "
                    "target write %d)\n",
                    m_id, curr_prio, packetPriorities[m_id][curr_prio],
                    tgt_prio, packetPriorities[m_id][tgt_prio],
                    readQueueSizes[curr_prio],
                    writeQueueSizes[curr_prio], readQueueSizes[tgt_prio],
                    writeQueueSizes[tgt_prio]);

            // Check both read and write queue
            for (auto q : queues) {
                escalateQueues(*q, queue_entry_size, m_id,
                               curr_prio, tgt_prio);
            }
        }
    }

    DPRINTF(QOS,
            "QoSMemCtrl::escalate Completed master %s [id %d] to priority %d "
            "(now %d packets)\n\t(total read %d, total write %d)\n",
            masters[m_id], m_id, tgt_prio, packetPriorities[m_id][tgt_prio],
            readQueueSizes[tgt_prio], writeQueueSizes[tgt_prio]);
}

template<typename Queues>
uint8_t
MemCtrl::qosSchedule(std::initializer_list<Queues*> queues,
                     const uint64_t queue_entry_size,
                     const PacketPtr pkt)
{
    // Schedule packet.
    uint8_t pkt_priority = schedule(pkt);

    assert(pkt_priority < numPriorities());

    pkt->qosValue(pkt_priority);

    if (qosSyncroScheduler) {
        // Call the scheduling function on all other masters.
        for (const auto& m : masters) {

            if (m.first == pkt->masterId())
                continue;

            uint8_t prio = schedule(m.first, 0);

            if (qosPriorityEscalation) {
                DPRINTF(QOS,
                        "QoSMemCtrl::qosSchedule: (syncro) escalating "
                        "MASTER %s to assigned priority %d\n",
                        _system->getMasterName(m.first),
                        prio);
                escalate(queues, queue_entry_size, m.first, prio);
            }
        }
    }

    if (qosPriorityEscalation) {
        DPRINTF(QOS,
                "QoSMemCtrl::qosSchedule: escalating "
                "MASTER %s to assigned priority %d\n",
                _system->getMasterName(pkt->masterId()),
                pkt_priority);
        escalate(queues, queue_entry_size, pkt->masterId(), pkt_priority);
    }

    // Update last service tick for selected priority
    serviceTick[pkt_priority] = curTick();

    return pkt_priority;
}

} // namespace QoS

#endif /* __MEM_QOS_MEM_CTRL_HH__ */